You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by be...@apache.org on 2015/08/17 11:20:37 UTC

[1/7] cassandra git commit: Improve transaction log under FS corruption

Repository: cassandra
Updated Branches:
  refs/heads/cassandra-3.0 47b66d7c1 -> 5726625a5
  refs/heads/trunk 5e9835ba7 -> 13ec86ce9


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
index 14db2d1..e0786f9 100644
--- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
+++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
@@ -23,7 +23,6 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
-import com.google.common.collect.Lists;
 import org.apache.commons.lang3.StringUtils;
 
 import org.junit.AfterClass;
@@ -236,7 +235,7 @@ public class DirectoriesTest
             Set<File> listed;
 
             // List all but no snapshot, backup
-            lister = directories.sstableLister();
+            lister = directories.sstableLister(Directories.OnTxnErr.THROW);
             listed = new HashSet<>(lister.listFiles());
             for (File f : files.get(cfm.cfName))
             {
@@ -247,7 +246,7 @@ public class DirectoriesTest
             }
 
             // List all but including backup (but no snapshot)
-            lister = directories.sstableLister().includeBackups(true);
+            lister = directories.sstableLister(Directories.OnTxnErr.THROW).includeBackups(true);
             listed = new HashSet<>(lister.listFiles());
             for (File f : files.get(cfm.cfName))
             {
@@ -258,7 +257,7 @@ public class DirectoriesTest
             }
 
             // Skip temporary and compacted
-            lister = directories.sstableLister().skipTemporary(true);
+            lister = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
             listed = new HashSet<>(lister.listFiles());
             for (File f : files.get(cfm.cfName))
             {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
index 30812d2..aedba0e 100644
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
@@ -37,6 +37,7 @@ import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.KeyspaceParams;
@@ -163,10 +164,7 @@ public class KeyCacheTest
 
         refs.release();
 
-        while (ScheduledExecutors.nonPeriodicTasks.getActiveCount() + ScheduledExecutors.nonPeriodicTasks.getQueue().size() > 0)
-        {
-            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);;
-        }
+        TransactionLog.waitForDeletions();
 
         // after releasing the reference this should drop to 2
         assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index cc4038d..8889488 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -43,7 +43,7 @@ import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.compaction.Scrubber;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.db.partitions.Partition;
@@ -371,7 +371,7 @@ public class ScrubTest
             {
                 scrubber.scrub();
             }
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             cfs.loadNewSSTables();
             assertOrderedAll(cfs, 7);
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
index e9c903e..7f1b2bd 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
@@ -32,14 +32,9 @@ import org.junit.Test;
 
 import junit.framework.Assert;
 import org.apache.cassandra.MockSchema;
-import org.apache.cassandra.Util;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.big.BigTableReader;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.Refs;
 
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertNotNull;
@@ -164,10 +159,10 @@ public class HelpersTest
     public void testMarkObsolete()
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
-        TransactionLogs txnLogs = new TransactionLogs(OperationType.UNKNOWN, cfs.metadata);
+        TransactionLog txnLogs = new TransactionLog(OperationType.UNKNOWN, cfs.metadata);
         Iterable<SSTableReader> readers = Lists.newArrayList(MockSchema.sstable(1, cfs), MockSchema.sstable(2, cfs));
 
-        List<TransactionLogs.Obsoletion> obsoletions = new ArrayList<>();
+        List<TransactionLog.Obsoletion> obsoletions = new ArrayList<>();
         Assert.assertNull(Helpers.prepareForObsoletion(readers, txnLogs, obsoletions, null));
         assertNotNull(obsoletions);
         assertEquals(2, obsoletions.size());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
index a376a61..db27662 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
@@ -249,7 +249,7 @@ public class LifecycleTransactionTest extends AbstractTransactionalTest
 
     protected TestableTransaction newTest()
     {
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         SSTableReader.resetTidying();
         return new TxnTest();
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
index 44f4d30..309e35a 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
@@ -89,10 +89,9 @@ public class RealTransactionsTest extends SchemaLoader
         SSTableReader oldSSTable = getSSTable(cfs, 1);
         LifecycleTransaction txn = cfs.getTracker().tryModify(oldSSTable, OperationType.COMPACTION);
         SSTableReader newSSTable = replaceSSTable(cfs, txn, false);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
-        assertFiles(txn.logs().getDataFolder(), new HashSet<>(newSSTable.getAllFilePaths()));
-        assertFiles(txn.logs().getLogsFolder(), Collections.<String>emptySet());
+        assertFiles(txn.log().getDataFolder(), new HashSet<>(newSSTable.getAllFilePaths()));
     }
 
     @Test
@@ -105,10 +104,9 @@ public class RealTransactionsTest extends SchemaLoader
         LifecycleTransaction txn = cfs.getTracker().tryModify(oldSSTable, OperationType.COMPACTION);
 
         replaceSSTable(cfs, txn, true);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
-        assertFiles(txn.logs().getDataFolder(), new HashSet<>(oldSSTable.getAllFilePaths()));
-        assertFiles(txn.logs().getLogsFolder(), Collections.<String>emptySet());
+        assertFiles(txn.log().getDataFolder(), new HashSet<>(oldSSTable.getAllFilePaths()));
     }
 
     @Test
@@ -120,11 +118,6 @@ public class RealTransactionsTest extends SchemaLoader
         SSTableReader ssTableReader = getSSTable(cfs, 100);
 
         String dataFolder = cfs.getLiveSSTables().iterator().next().descriptor.directory.getPath();
-        String transactionLogsFolder = StringUtils.join(dataFolder, File.separator, Directories.TRANSACTIONS_SUBDIR);
-
-        assertTrue(new File(transactionLogsFolder).exists());
-        assertFiles(transactionLogsFolder, Collections.<String>emptySet());
-
         assertFiles(dataFolder, new HashSet<>(ssTableReader.getAllFilePaths()));
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index ea0d9a8..3a943c4 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -187,9 +187,9 @@ public class TrackerTest
     public void testDropSSTables()
     {
         testDropSSTables(false);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         testDropSSTables(true);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
     }
 
     private void testDropSSTables(boolean invalidate)
@@ -203,62 +203,54 @@ public class TrackerTest
                                                              MockSchema.sstable(2, 71, true, cfs));
         tracker.addInitialSSTables(copyOf(readers));
 
-        try
+        try (LifecycleTransaction txn = tracker.tryModify(readers.get(0), OperationType.COMPACTION))
         {
-           // TransactionLogs.pauseDeletions(true);
-            try (LifecycleTransaction txn = tracker.tryModify(readers.get(0), OperationType.COMPACTION))
+            if (invalidate)
             {
-                if (invalidate)
-                {
-                    cfs.invalidate(false);
-                }
-                else
-                {
-                    tracker.dropSSTables();
-                    TransactionLogs.waitForDeletions();
-                }
-                Assert.assertEquals(9, cfs.metric.totalDiskSpaceUsed.getCount());
-                Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
-                Assert.assertEquals(1, tracker.getView().sstables.size());
+                cfs.invalidate(false);
             }
-            if (!invalidate)
+            else
             {
-                Assert.assertEquals(1, tracker.getView().sstables.size());
-                Assert.assertEquals(readers.get(0), Iterables.getFirst(tracker.getView().sstables, null));
-                Assert.assertEquals(1, readers.get(0).selfRef().globalCount());
-                Assert.assertFalse(readers.get(0).isMarkedCompacted());
-                for (SSTableReader reader : readers.subList(1, 3))
-                {
-                    Assert.assertEquals(0, reader.selfRef().globalCount());
-                    Assert.assertTrue(reader.isMarkedCompacted());
-                }
-
-                Assert.assertNull(tracker.dropSSTables(reader -> reader != readers.get(0), OperationType.UNKNOWN, null));
-
-                Assert.assertEquals(1, tracker.getView().sstables.size());
-                Assert.assertEquals(3, listener.received.size());
-                Assert.assertEquals(tracker, listener.senders.get(0));
-                Assert.assertTrue(listener.received.get(0) instanceof SSTableDeletingNotification);
-                Assert.assertTrue(listener.received.get(1) instanceof  SSTableDeletingNotification);
-                Assert.assertTrue(listener.received.get(2) instanceof SSTableListChangedNotification);
-                Assert.assertEquals(readers.get(1), ((SSTableDeletingNotification) listener.received.get(0)).deleting);
-                Assert.assertEquals(readers.get(2), ((SSTableDeletingNotification)listener.received.get(1)).deleting);
-                Assert.assertEquals(2, ((SSTableListChangedNotification) listener.received.get(2)).removed.size());
-                Assert.assertEquals(0, ((SSTableListChangedNotification) listener.received.get(2)).added.size());
-                Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
-                readers.get(0).selfRef().release();
+                tracker.dropSSTables();
+                TransactionLog.waitForDeletions();
             }
-            else
+            Assert.assertEquals(9, cfs.metric.totalDiskSpaceUsed.getCount());
+            Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
+            Assert.assertEquals(1, tracker.getView().sstables.size());
+        }
+        if (!invalidate)
+        {
+            Assert.assertEquals(1, tracker.getView().sstables.size());
+            Assert.assertEquals(readers.get(0), Iterables.getFirst(tracker.getView().sstables, null));
+            Assert.assertEquals(1, readers.get(0).selfRef().globalCount());
+            Assert.assertFalse(readers.get(0).isMarkedCompacted());
+            for (SSTableReader reader : readers.subList(1, 3))
             {
-                Assert.assertEquals(0, tracker.getView().sstables.size());
-                Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
-                for (SSTableReader reader : readers)
-                    Assert.assertTrue(reader.isMarkedCompacted());
+                Assert.assertEquals(0, reader.selfRef().globalCount());
+                Assert.assertTrue(reader.isMarkedCompacted());
             }
+
+            Assert.assertNull(tracker.dropSSTables(reader -> reader != readers.get(0), OperationType.UNKNOWN, null));
+
+            Assert.assertEquals(1, tracker.getView().sstables.size());
+            Assert.assertEquals(3, listener.received.size());
+            Assert.assertEquals(tracker, listener.senders.get(0));
+            Assert.assertTrue(listener.received.get(0) instanceof SSTableDeletingNotification);
+            Assert.assertTrue(listener.received.get(1) instanceof  SSTableDeletingNotification);
+            Assert.assertTrue(listener.received.get(2) instanceof SSTableListChangedNotification);
+            Assert.assertEquals(readers.get(1), ((SSTableDeletingNotification) listener.received.get(0)).deleting);
+            Assert.assertEquals(readers.get(2), ((SSTableDeletingNotification)listener.received.get(1)).deleting);
+            Assert.assertEquals(2, ((SSTableListChangedNotification) listener.received.get(2)).removed.size());
+            Assert.assertEquals(0, ((SSTableListChangedNotification) listener.received.get(2)).added.size());
+            Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
+            readers.get(0).selfRef().release();
         }
-        finally
+        else
         {
-           // TransactionLogs.pauseDeletions(false);
+            Assert.assertEquals(0, tracker.getView().sstables.size());
+            Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
+            for (SSTableReader reader : readers)
+                Assert.assertTrue(reader.isMarkedCompacted());
         }
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogTest.java
new file mode 100644
index 0000000..7739163
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogTest.java
@@ -0,0 +1,791 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import junit.framework.Assert;
+import org.apache.cassandra.MockSchema;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.*;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.BufferedSegmentedFile;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SegmentedFile;
+import org.apache.cassandra.utils.AlwaysPresentFilter;
+import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+public class TransactionLogTest extends AbstractTransactionalTest
+{
+    private static final String KEYSPACE = "TransactionLogsTest";
+
+    @BeforeClass
+    public static void setUp()
+    {
+        MockSchema.cleanup();
+    }
+
+    protected AbstractTransactionalTest.TestableTransaction newTest() throws Exception
+    {
+        TransactionLog.waitForDeletions();
+        SSTableReader.resetTidying();
+        return new TxnTest();
+    }
+
+    private static final class TxnTest extends TestableTransaction
+    {
+        private final static class Transaction extends Transactional.AbstractTransactional implements Transactional
+        {
+            final ColumnFamilyStore cfs;
+            final TransactionLog txnLogs;
+            final SSTableReader sstableOld;
+            final SSTableReader sstableNew;
+            final TransactionLog.SSTableTidier tidier;
+
+            public Transaction(ColumnFamilyStore cfs, TransactionLog txnLogs) throws IOException
+            {
+                this.cfs = cfs;
+                this.txnLogs = txnLogs;
+                this.sstableOld = sstable(cfs, 0, 128);
+                this.sstableNew = sstable(cfs, 1, 128);
+
+                assertNotNull(txnLogs);
+                assertNotNull(txnLogs.getId());
+                Assert.assertEquals(OperationType.COMPACTION, txnLogs.getType());
+
+                txnLogs.trackNew(sstableNew);
+                tidier = txnLogs.obsoleted(sstableOld);
+                assertNotNull(tidier);
+            }
+
+            protected Throwable doCommit(Throwable accumulate)
+            {
+                sstableOld.markObsolete(tidier);
+                sstableOld.selfRef().release();
+                TransactionLog.waitForDeletions();
+
+                Throwable ret = txnLogs.commit(accumulate);
+
+                sstableNew.selfRef().release();
+                return ret;
+            }
+
+            protected Throwable doAbort(Throwable accumulate)
+            {
+                tidier.abort();
+                TransactionLog.waitForDeletions();
+
+                Throwable ret = txnLogs.abort(accumulate);
+
+                sstableNew.selfRef().release();
+                sstableOld.selfRef().release();
+                return ret;
+            }
+
+            protected void doPrepare()
+            {
+                txnLogs.prepareToCommit();
+            }
+
+            protected void assertInProgress() throws Exception
+            {
+                assertFiles(txnLogs.getDataFolder(), Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths(),
+                                                                                      sstableOld.getAllFilePaths(),
+                                                                                      Collections.singleton(txnLogs.getData().getLogFile().file.getPath()))));
+            }
+
+            protected void assertPrepared() throws Exception
+            {
+            }
+
+            protected void assertAborted() throws Exception
+            {
+                assertFiles(txnLogs.getDataFolder(), new HashSet<>(sstableOld.getAllFilePaths()));
+            }
+
+            protected void assertCommitted() throws Exception
+            {
+                assertFiles(txnLogs.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
+            }
+        }
+
+        final Transaction txn;
+
+        private TxnTest() throws IOException
+        {
+            this(MockSchema.newCFS(KEYSPACE));
+        }
+
+        private TxnTest(ColumnFamilyStore cfs) throws IOException
+        {
+            this(cfs, new TransactionLog(OperationType.COMPACTION, cfs.metadata));
+        }
+
+        private TxnTest(ColumnFamilyStore cfs, TransactionLog txnLogs) throws IOException
+        {
+            this(new Transaction(cfs, txnLogs));
+        }
+
+        private TxnTest(Transaction txn)
+        {
+            super(txn);
+            this.txn = txn;
+        }
+
+        protected void assertInProgress() throws Exception
+        {
+            txn.assertInProgress();
+        }
+
+        protected void assertPrepared() throws Exception
+        {
+            txn.assertPrepared();
+        }
+
+        protected void assertAborted() throws Exception
+        {
+            txn.assertAborted();
+        }
+
+        protected void assertCommitted() throws Exception
+        {
+            txn.assertCommitted();
+        }
+    }
+
+    @Test
+    public void testUntrack() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        // complete a transaction without keep the new files since they were untracked
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        transactionLog.untrackNew(sstableNew);
+
+        transactionLog.finish();
+
+        sstableNew.selfRef().release();
+        Thread.sleep(1);
+        TransactionLog.waitForDeletions();
+
+        assertFiles(transactionLog.getDataFolder(), Collections.<String>emptySet());
+    }
+
+    @Test
+    public void testCommitSameDesc() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld1 = sstable(cfs, 0, 128);
+        SSTableReader sstableOld2 = sstable(cfs, 0, 256);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+
+        sstableOld1.setReplaced();
+
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstableOld2);
+        assertNotNull(tidier);
+
+        transactionLog.finish();
+
+        sstableOld2.markObsolete(tidier);
+
+        sstableOld1.selfRef().release();
+        sstableOld2.selfRef().release();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
+
+        sstableNew.selfRef().release();
+    }
+
+    @Test
+    public void testCommitOnlyNew() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstable);
+        transactionLog.finish();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstable.getAllFilePaths()));
+
+        sstable.selfRef().release();
+    }
+
+    @Test
+    public void testCommitOnlyOld() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstable);
+        assertNotNull(tidier);
+
+        transactionLog.finish();
+        sstable.markObsolete(tidier);
+        sstable.selfRef().release();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>());
+    }
+
+    @Test
+    public void testAbortOnlyNew() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstable);
+        transactionLog.abort();
+
+        sstable.selfRef().release();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>());
+    }
+
+    @Test
+    public void testAbortOnlyOld() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstable);
+        assertNotNull(tidier);
+
+        tidier.abort();
+        transactionLog.abort();
+
+        sstable.selfRef().release();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstable.getAllFilePaths()));
+    }
+
+    @Test
+    public void testRemoveUnfinishedLeftovers_abort() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld = sstable(cfs, 0, 128);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        // simulate tracking sstables with a failed transaction (new log file NOT deleted)
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstableOld);
+
+        Set<File> tmpFiles = Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths().stream().map(p -> new File(p)).collect(Collectors.toList()),
+                                                              Collections.singleton(transactionLog.getData().getLogFile().file)));
+
+        sstableNew.selfRef().release();
+        sstableOld.selfRef().release();
+
+        Assert.assertEquals(tmpFiles, TransactionLog.getTemporaryFiles(cfs.metadata, sstableNew.descriptor.directory));
+
+        // normally called at startup
+        TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+        // sstableOld should be only table left
+        Directories directories = new Directories(cfs.metadata);
+        Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
+        assertEquals(1, sstables.size());
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstableOld.getAllFilePaths()));
+
+        tidier.run();
+
+        // complete the transaction to avoid LEAK errors
+        transactionLog.close();
+    }
+
+    @Test
+    public void testRemoveUnfinishedLeftovers_commit() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld = sstable(cfs, 0, 128);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        // simulate tracking sstables with a committed transaction (new log file deleted)
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstableOld);
+
+        //Fake a commit
+        transactionLog.getData().getLogFile().commit();
+
+        Set<File> tmpFiles = Sets.newHashSet(Iterables.concat(sstableOld.getAllFilePaths().stream().map(p -> new File(p)).collect(Collectors.toList()),
+                                                              Collections.singleton(transactionLog.getData().getLogFile().file)));
+
+        sstableNew.selfRef().release();
+        sstableOld.selfRef().release();
+
+        Assert.assertEquals(tmpFiles, TransactionLog.getTemporaryFiles(cfs.metadata, sstableOld.descriptor.directory));
+
+        // normally called at startup
+        TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+        // sstableNew should be only table left
+        Directories directories = new Directories(cfs.metadata);
+        Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
+        assertEquals(1, sstables.size());
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
+
+        tidier.run();
+
+        // complete the transaction to avoid LEAK errors
+        assertNull(transactionLog.complete(null));
+    }
+
+    @Test
+    public void testGetTemporaryFiles() throws IOException
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable1 = sstable(cfs, 0, 128);
+
+        File dataFolder = sstable1.descriptor.directory;
+
+        Set<File> tmpFiles = TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder);
+        assertNotNull(tmpFiles);
+        assertEquals(0, tmpFiles.size());
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.WRITE, cfs.metadata);
+        Directories directories = new Directories(cfs.metadata);
+
+        File[] beforeSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
+
+        SSTableReader sstable2 = sstable(cfs, 1, 128);
+        transactionLog.trackNew(sstable2);
+
+        Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
+        assertEquals(2, sstables.size());
+
+        // this should contain sstable1, sstable2 and the transaction log file
+        File[] afterSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
+
+        int numNewFiles = afterSecondSSTable.length - beforeSecondSSTable.length;
+        assertEquals(numNewFiles - 1, sstable2.getAllFilePaths().size()); // new files except for transaction log file
+
+        tmpFiles = TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder);
+        assertNotNull(tmpFiles);
+        assertEquals(numNewFiles, tmpFiles.size());
+
+        File ssTable2DataFile = new File(sstable2.descriptor.filenameFor(Component.DATA));
+        File ssTable2IndexFile = new File(sstable2.descriptor.filenameFor(Component.PRIMARY_INDEX));
+
+        assertTrue(tmpFiles.contains(ssTable2DataFile));
+        assertTrue(tmpFiles.contains(ssTable2IndexFile));
+
+        List<File> files = directories.sstableLister(Directories.OnTxnErr.THROW).listFiles();
+        List<File> filesNoTmp = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
+        assertNotNull(files);
+        assertNotNull(filesNoTmp);
+
+        assertTrue(files.contains(ssTable2DataFile));
+        assertTrue(files.contains(ssTable2IndexFile));
+
+        assertFalse(filesNoTmp.contains(ssTable2DataFile));
+        assertFalse(filesNoTmp.contains(ssTable2IndexFile));
+
+        transactionLog.finish();
+
+        //Now it should be empty since the transaction has finished
+        tmpFiles = TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder);
+        assertNotNull(tmpFiles);
+        assertEquals(0, tmpFiles.size());
+
+        filesNoTmp = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
+        assertNotNull(filesNoTmp);
+        assertTrue(filesNoTmp.contains(ssTable2DataFile));
+        assertTrue(filesNoTmp.contains(ssTable2IndexFile));
+
+        sstable1.selfRef().release();
+        sstable2.selfRef().release();
+    }
+
+    @Test
+    public void testWrongChecksumLastLine() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake a commit with invalid checksum
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+                          },
+                          true);
+    }
+
+    @Test
+    public void testWrongChecksumSecondFromLastLine() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake two lines with invalid checksum
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("add:[ma-3-big,%d,4][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+                          },
+                          false);
+    }
+
+    @Test
+    public void testWrongChecksumLastLineMissingFile() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake a commit with invalid checksum and also delete one of the old files
+                              for (String filePath : s.getAllFilePaths())
+                              {
+                                  if (filePath.endsWith("Data.db"))
+                                  {
+                                      FileUtils.delete(filePath);
+                                      break;
+                                  }
+                              }
+
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+                          },
+                          false);
+    }
+
+    @Test
+    public void testWrongChecksumLastLineWrongRecordFormat() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake a commit with invalid checksum and a wrong record format (extra spaces)
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d ,0 ,0 ][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+                          },
+                          true);
+    }
+
+    @Test
+    public void testMissingChecksumLastLine() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          {
+                              // Fake a commit without a checksum
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0]",
+                                                             System.currentTimeMillis()));
+                          },
+                          true);
+    }
+
+    @Test
+    public void testMissingChecksumSecondFromLastLine() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake two lines without a checksum
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("add:[ma-3-big,%d,4]",
+                                                             System.currentTimeMillis()));
+
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0]",
+                                                             System.currentTimeMillis()));
+                          },
+                          false);
+    }
+
+    private void testCorruptRecord(BiConsumer<TransactionLog, SSTableReader> modifier, boolean isRecoverable) throws IOException
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld = sstable(cfs, 0, 128);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        File dataFolder = sstableOld.descriptor.directory;
+
+        // simulate tracking sstables with a committed transaction except the checksum will be wrong
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        transactionLog.obsoleted(sstableOld);
+
+        //Modify the transaction log in some way
+        modifier.accept(transactionLog, sstableOld);
+
+        String txnFilePath = transactionLog.getData().getLogFile().file.getPath();
+
+        transactionLog.complete(null);
+
+        sstableOld.selfRef().release();
+        sstableNew.selfRef().release();
+
+        if (isRecoverable)
+        { // the corruption is recoverable, we assume there is a commit record
+
+            //This should return the old files and the tx log
+            assertFiles(Iterables.concat(sstableOld.getAllFilePaths(), Collections.singleton(txnFilePath)),
+                        TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder));
+
+            //This should remove old files
+            TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+            assertFiles(dataFolder.getPath(), Sets.newHashSet(sstableNew.getAllFilePaths()));
+        }
+        else
+        { // if an intermediate line was modified, we cannot tell,
+          // it should just throw and handle the exception with a log message
+
+            //This should not return any files
+            assertEquals(Collections.emptyList(), new TransactionLog.FileLister(dataFolder.toPath(),
+                                                                                (file, type) -> type != Directories.FileType.FINAL,
+                                                                                Directories.OnTxnErr.IGNORE).list());
+
+            try
+            {
+                //This should throw a RuntimeException
+                new TransactionLog.FileLister(dataFolder.toPath(),
+                                              (file, type) -> type != Directories.FileType.FINAL,
+                                              Directories.OnTxnErr.THROW).list();
+                fail("Expected exception");
+            }
+            catch (RuntimeException ex)
+            {
+                // pass
+                ex.printStackTrace();
+            }
+
+            //This should not remove any files
+            TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+            assertFiles(dataFolder.getPath(), Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths(),
+                                                                               sstableOld.getAllFilePaths(),
+                                                                               Collections.singleton(txnFilePath))),
+                        true);
+        }
+    }
+
+    @Test
+    public void testObsoletedDataFileUpdateTimeChanged() throws IOException
+    {
+        testObsoletedFilesChanged(sstable ->
+                                 {
+                                     // increase the modification time of the Data file
+                                     for (String filePath : sstable.getAllFilePaths())
+                                     {
+                                         if (filePath.endsWith("Data.db"))
+                                             assertTrue(new File(filePath).setLastModified(System.currentTimeMillis() + 60000)); //one minute later
+                                     }
+                                 });
+    }
+
+    private void testObsoletedFilesChanged(Consumer<SSTableReader> modifier) throws IOException
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld = sstable(cfs, 0, 128);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        // simulate tracking sstables with a committed transaction except the checksum will be wrong
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        /*TransactionLog.SSTableTidier tidier =*/ transactionLog.obsoleted(sstableOld);
+
+        //modify the old sstable files
+        modifier.accept(sstableOld);
+
+        //Fake a commit
+        transactionLog.getData().getLogFile().commit();
+
+        //This should not remove the old files
+        TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+        assertFiles(transactionLog.getDataFolder(), Sets.newHashSet(Iterables.concat(
+                                                                                    sstableNew.getAllFilePaths(),
+                                                                                    sstableOld.getAllFilePaths(),
+                                                                                    Collections.singleton(transactionLog.getData().getLogFile().file.getPath()))));
+
+        sstableOld.selfRef().release();
+        sstableNew.selfRef().release();
+
+        // complete the transaction to avoid LEAK errors
+        assertNull(transactionLog.complete(null));
+
+        assertFiles(transactionLog.getDataFolder(), Sets.newHashSet(Iterables.concat(
+                                                                                    sstableNew.getAllFilePaths(),
+                                                                                    sstableOld.getAllFilePaths(),
+                                                                                    Collections.singleton(transactionLog.getData().getLogFile().file.getPath()))));
+    }
+
+    @Test
+    public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+        File dataFolder = sstable.descriptor.directory;
+
+        TransactionLog transactionLogs = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLogs);
+
+        TransactionLog.SSTableTidier tidier = transactionLogs.obsoleted(sstable);
+
+        transactionLogs.finish();
+        sstable.markObsolete(tidier);
+        sstable.selfRef().release();
+
+        for (int i = 0; i < 1000; i++)
+        {
+            // This should race with the asynchronous deletion of txn log files
+            // It doesn't matter what it returns but it should not throw
+            TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder);
+        }
+    }
+
+    private static SSTableReader sstable(ColumnFamilyStore cfs, int generation, int size) throws IOException
+    {
+        Directories dir = new Directories(cfs.metadata);
+        Descriptor descriptor = new Descriptor(dir.getDirectoryForNewSSTables(), cfs.keyspace.getName(), cfs.getColumnFamilyName(), generation);
+        Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
+        for (Component component : components)
+        {
+            File file = new File(descriptor.filenameFor(component));
+            file.createNewFile();
+            try (RandomAccessFile raf = new RandomAccessFile(file, "rw"))
+            {
+                raf.setLength(size);
+            }
+        }
+
+        SegmentedFile dFile = new BufferedSegmentedFile(new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))), RandomAccessReader.DEFAULT_BUFFER_SIZE, 0);
+        SegmentedFile iFile = new BufferedSegmentedFile(new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))), RandomAccessReader.DEFAULT_BUFFER_SIZE, 0);
+
+        SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.EMPTY_LIST);
+        StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata.comparator)
+                                                 .finalizeMetadata(cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
+                                                 .get(MetadataType.STATS);
+        SSTableReader reader = SSTableReader.internalOpen(descriptor,
+                                                          components,
+                                                          cfs.metadata,
+                                                          dFile,
+                                                          iFile,
+                                                          MockSchema.indexSummary.sharedCopy(),
+                                                          new AlwaysPresentFilter(),
+                                                          1L,
+                                                          metadata,
+                                                          SSTableReader.OpenReason.NORMAL,
+                                                          header);
+        reader.first = reader.last = MockSchema.readerBounds(generation);
+        return reader;
+    }
+
+    private static void assertFiles(String dirPath, Set<String> expectedFiles)
+    {
+        assertFiles(dirPath, expectedFiles, false);
+    }
+
+    private static void assertFiles(String dirPath, Set<String> expectedFiles, boolean excludeNonExistingFiles)
+    {
+        TransactionLog.waitForDeletions();
+
+        File dir = new File(dirPath);
+        for (File file : dir.listFiles())
+        {
+            if (file.isDirectory())
+                continue;
+
+            String filePath = file.getPath();
+            assertTrue(filePath, expectedFiles.contains(filePath));
+            expectedFiles.remove(filePath);
+        }
+
+        if (excludeNonExistingFiles)
+        {
+            for (String filePath : expectedFiles)
+            {
+                File file = new File(filePath);
+                if (!file.exists())
+                    expectedFiles.remove(filePath);
+            }
+        }
+
+        assertTrue(expectedFiles.toString(), expectedFiles.isEmpty());
+    }
+
+    private static void assertFiles(Iterable<String> filePaths, Set<File> expectedFiles)
+    {
+        for (String filePath : filePaths)
+        {
+            File file = new File(filePath);
+            assertTrue(filePath, expectedFiles.contains(file));
+            expectedFiles.remove(file);
+        }
+
+        assertTrue(expectedFiles.isEmpty());
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogsTest.java
deleted file mode 100644
index 991eed3..0000000
--- a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogsTest.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.lifecycle;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
-import java.util.*;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertNotNull;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.compaction.*;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.BufferedSegmentedFile;
-import org.apache.cassandra.io.util.ChannelProxy;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.io.util.SegmentedFile;
-import org.apache.cassandra.utils.AlwaysPresentFilter;
-import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
-import org.apache.cassandra.utils.concurrent.Transactional;
-
-public class TransactionLogsTest extends AbstractTransactionalTest
-{
-    private static final String KEYSPACE = "TransactionLogsTest";
-
-    @BeforeClass
-    public static void setUp()
-    {
-        MockSchema.cleanup();
-    }
-
-    protected AbstractTransactionalTest.TestableTransaction newTest() throws Exception
-    {
-        TransactionLogs.waitForDeletions();
-        SSTableReader.resetTidying();
-        return new TxnTest();
-    }
-
-    private static final class TxnTest extends TestableTransaction
-    {
-        private final static class Transaction extends Transactional.AbstractTransactional implements Transactional
-        {
-            final ColumnFamilyStore cfs;
-            final TransactionLogs txnLogs;
-            final SSTableReader sstableOld;
-            final SSTableReader sstableNew;
-            final TransactionLogs.SSTableTidier tidier;
-
-            public Transaction(ColumnFamilyStore cfs, TransactionLogs txnLogs) throws IOException
-            {
-                this.cfs = cfs;
-                this.txnLogs = txnLogs;
-                this.sstableOld = sstable(cfs, 0, 128);
-                this.sstableNew = sstable(cfs, 1, 128);
-
-                assertNotNull(txnLogs);
-                assertNotNull(txnLogs.getId());
-                Assert.assertEquals(OperationType.COMPACTION, txnLogs.getType());
-
-                txnLogs.trackNew(sstableNew);
-                tidier = txnLogs.obsoleted(sstableOld);
-                assertNotNull(tidier);
-            }
-
-            protected Throwable doCommit(Throwable accumulate)
-            {
-                sstableOld.markObsolete(tidier);
-                sstableOld.selfRef().release();
-                TransactionLogs.waitForDeletions();
-
-                Throwable ret = txnLogs.commit(accumulate);
-
-                sstableNew.selfRef().release();
-                return ret;
-            }
-
-            protected Throwable doAbort(Throwable accumulate)
-            {
-                tidier.abort();
-                TransactionLogs.waitForDeletions();
-
-                Throwable ret = txnLogs.abort(accumulate);
-
-                sstableNew.selfRef().release();
-                sstableOld.selfRef().release();
-                return ret;
-            }
-
-            protected void doPrepare()
-            {
-                txnLogs.prepareToCommit();
-            }
-
-            protected void assertInProgress() throws Exception
-            {
-                assertFiles(txnLogs.getDataFolder(), Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths(),
-                                                                                      sstableOld.getAllFilePaths())));
-                assertFiles(txnLogs.getLogsFolder(), Sets.newHashSet(txnLogs.getData().oldLog().file.getPath(),
-                                                                     txnLogs.getData().newLog().file.getPath()));
-                assertEquals(2, TransactionLogs.getLogFiles(cfs.metadata).size());
-            }
-
-            protected void assertPrepared() throws Exception
-            {
-            }
-
-            protected void assertAborted() throws Exception
-            {
-                assertFiles(txnLogs.getDataFolder(), new HashSet<>(sstableOld.getAllFilePaths()));
-                assertFiles(txnLogs.getLogsFolder(), Collections.<String>emptySet());
-                assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-            }
-
-            protected void assertCommitted() throws Exception
-            {
-                assertFiles(txnLogs.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
-                assertFiles(txnLogs.getLogsFolder(), Collections.<String>emptySet());
-                assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-            }
-        }
-
-        final Transaction txn;
-
-        private TxnTest() throws IOException
-        {
-            this(MockSchema.newCFS(KEYSPACE));
-        }
-
-        private TxnTest(ColumnFamilyStore cfs) throws IOException
-        {
-            this(cfs, new TransactionLogs(OperationType.COMPACTION, cfs.metadata));
-        }
-
-        private TxnTest(ColumnFamilyStore cfs, TransactionLogs txnLogs) throws IOException
-        {
-            this(new Transaction(cfs, txnLogs));
-        }
-
-        private TxnTest(Transaction txn)
-        {
-            super(txn);
-            this.txn = txn;
-        }
-
-        protected void assertInProgress() throws Exception
-        {
-            txn.assertInProgress();
-        }
-
-        protected void assertPrepared() throws Exception
-        {
-            txn.assertPrepared();
-        }
-
-        protected void assertAborted() throws Exception
-        {
-            txn.assertAborted();
-        }
-
-        protected void assertCommitted() throws Exception
-        {
-            txn.assertCommitted();
-        }
-    }
-
-    @Test
-    public void testUntrack() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstableNew = sstable(cfs, 1, 128);
-
-        // complete a transaction without keep the new files since they were untracked
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstableNew);
-        transactionLogs.untrackNew(sstableNew);
-
-        transactionLogs.finish();
-
-        assertFiles(transactionLogs.getDataFolder(), Collections.<String>emptySet());
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-
-        sstableNew.selfRef().release();
-    }
-
-    @Test
-    public void testCommitSameDesc() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstableOld1 = sstable(cfs, 0, 128);
-        SSTableReader sstableOld2 = sstable(cfs, 0, 256);
-        SSTableReader sstableNew = sstable(cfs, 1, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstableNew);
-
-        sstableOld1.setReplaced();
-
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstableOld2);
-        assertNotNull(tidier);
-
-        transactionLogs.finish();
-
-        sstableOld2.markObsolete(tidier);
-
-        sstableOld1.selfRef().release();
-        sstableOld2.selfRef().release();
-
-        TransactionLogs.waitForDeletions();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-
-        sstableNew.selfRef().release();
-    }
-
-    @Test
-    public void testCommitOnlyNew() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstable);
-        transactionLogs.finish();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstable.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-
-        sstable.selfRef().release();
-    }
-
-    @Test
-    public void testCommitOnlyOld() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstable);
-        assertNotNull(tidier);
-
-        transactionLogs.finish();
-        sstable.markObsolete(tidier);
-        sstable.selfRef().release();
-
-        TransactionLogs.waitForDeletions();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>());
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-    }
-
-    @Test
-    public void testAbortOnlyNew() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstable);
-        transactionLogs.abort();
-
-        sstable.selfRef().release();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>());
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-    }
-
-    @Test
-    public void testAbortOnlyOld() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstable);
-        assertNotNull(tidier);
-
-        tidier.abort();
-        transactionLogs.abort();
-
-        sstable.selfRef().release();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstable.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-    }
-
-    private File copyToTmpFile(File file) throws IOException
-    {
-        File ret = File.createTempFile(file.getName(), ".tmp");
-        ret.deleteOnExit();
-        Files.copy(file.toPath(), ret.toPath(), StandardCopyOption.REPLACE_EXISTING);
-        return ret;
-    }
-
-    @Test
-    public void testRemoveUnfinishedLeftovers_newLogFound() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstableOld = sstable(cfs, 0, 128);
-        SSTableReader sstableNew = sstable(cfs, 1, 128);
-
-        // simulate tracking sstables with a failed transaction (new log file NOT deleted)
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstableNew);
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstableOld);
-
-        File tmpNewLog = copyToTmpFile(transactionLogs.getData().newLog().file);
-        File tmpOldLog = copyToTmpFile(transactionLogs.getData().oldLog().file);
-
-        Set<File> tmpFiles = new HashSet<>(TransactionLogs.getLogFiles(cfs.metadata));
-        for (String p : sstableNew.getAllFilePaths())
-            tmpFiles.add(new File(p));
-
-        sstableNew.selfRef().release();
-        sstableOld.selfRef().release();
-
-        Assert.assertEquals(tmpFiles, TransactionLogs.getTemporaryFiles(cfs.metadata, sstableNew.descriptor.directory));
-
-        // normally called at startup
-        TransactionLogs.removeUnfinishedLeftovers(cfs.metadata);
-
-        // sstable should not have been removed because the new log was found
-        Directories directories = new Directories(cfs.metadata);
-        Map<Descriptor, Set<Component>> sstables = directories.sstableLister().list();
-        assertEquals(1, sstables.size());
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstableOld.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-
-        tidier.run();
-
-        // copy old transaction files contents back or transactionlogs will throw assertions
-        Files.move(tmpNewLog.toPath(), transactionLogs.getData().newLog().file.toPath());
-        Files.move(tmpOldLog.toPath(), transactionLogs.getData().oldLog().file.toPath());
-
-        transactionLogs.close();
-    }
-
-    @Test
-    public void testRemoveUnfinishedLeftovers_oldLogFound() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstableOld = sstable(cfs, 0, 128);
-        SSTableReader sstableNew = sstable(cfs, 1, 128);
-
-        // simulate tracking sstables with a committed transaction (new log file deleted)
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstableNew);
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstableOld);
-
-        File tmpNewLog = copyToTmpFile(transactionLogs.getData().newLog().file);
-        File tmpOldLog = copyToTmpFile(transactionLogs.getData().oldLog().file);
-
-        transactionLogs.getData().newLog().delete(false);
-
-        Set<File> tmpFiles = new HashSet<>(TransactionLogs.getLogFiles(cfs.metadata));
-        for (String p : sstableOld.getAllFilePaths())
-            tmpFiles.add(new File(p));
-
-        sstableNew.selfRef().release();
-        sstableOld.selfRef().release();
-
-        Assert.assertEquals(tmpFiles, TransactionLogs.getTemporaryFiles(cfs.metadata, sstableOld.descriptor.directory));
-
-        // normally called at startup
-        TransactionLogs.removeUnfinishedLeftovers(cfs.metadata);
-
-        // sstable should have been removed because there was no new log.
-        Directories directories = new Directories(cfs.metadata);
-        Map<Descriptor, Set<Component>> sstables = directories.sstableLister().list();
-        assertEquals(1, sstables.size());
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-
-        tidier.run();
-
-        // copy old transaction files contents back or transactionlogs will throw assertions
-        Files.move(tmpNewLog.toPath(), transactionLogs.getData().newLog().file.toPath());
-        Files.move(tmpOldLog.toPath(), transactionLogs.getData().oldLog().file.toPath());
-
-        transactionLogs.close();
-    }
-
-    @Test
-    public void testGetTemporaryFiles() throws IOException
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable1 = sstable(cfs, 0, 128);
-
-        File dataFolder = sstable1.descriptor.directory;
-
-        Set<File> tmpFiles = TransactionLogs.getTemporaryFiles(cfs.metadata, dataFolder);
-        assertNotNull(tmpFiles);
-        assertEquals(0, tmpFiles.size());
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.WRITE, cfs.metadata);
-        Directories directories = new Directories(cfs.metadata);
-
-        File[] beforeSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
-
-        SSTableReader sstable2 = sstable(cfs, 1, 128);
-        transactionLogs.trackNew(sstable2);
-
-        Map<Descriptor, Set<Component>> sstables = directories.sstableLister().list();
-        assertEquals(2, sstables.size());
-
-        File[] afterSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
-        int numNewFiles = afterSecondSSTable.length - beforeSecondSSTable.length;
-        assertTrue(numNewFiles == sstable2.getAllFilePaths().size());
-
-        tmpFiles = TransactionLogs.getTemporaryFiles(cfs.metadata, dataFolder);
-        assertNotNull(tmpFiles);
-        assertEquals(numNewFiles + 2, tmpFiles.size()); //the extra files are the transaction log files
-
-        File ssTable2DataFile = new File(sstable2.descriptor.filenameFor(Component.DATA));
-        File ssTable2IndexFile = new File(sstable2.descriptor.filenameFor(Component.PRIMARY_INDEX));
-
-        assertTrue(tmpFiles.contains(ssTable2DataFile));
-        assertTrue(tmpFiles.contains(ssTable2IndexFile));
-
-        List<File> files = directories.sstableLister().listFiles();
-        List<File> filesNoTmp = directories.sstableLister().skipTemporary(true).listFiles();
-        assertNotNull(files);
-        assertNotNull(filesNoTmp);
-
-        assertTrue(files.contains(ssTable2DataFile));
-        assertTrue(files.contains(ssTable2IndexFile));
-
-        assertFalse(filesNoTmp.contains(ssTable2DataFile));
-        assertFalse(filesNoTmp.contains(ssTable2IndexFile));
-
-        transactionLogs.finish();
-
-        //Now it should be empty since the transaction has finished
-        tmpFiles = TransactionLogs.getTemporaryFiles(cfs.metadata, dataFolder);
-        assertNotNull(tmpFiles);
-        assertEquals(0, tmpFiles.size());
-
-        filesNoTmp = directories.sstableLister().skipTemporary(true).listFiles();
-        assertNotNull(filesNoTmp);
-        assertTrue(filesNoTmp.contains(ssTable2DataFile));
-        assertTrue(filesNoTmp.contains(ssTable2IndexFile));
-
-        sstable1.selfRef().release();
-        sstable2.selfRef().release();
-    }
-
-    @Test
-    public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-        File dataFolder = sstable.descriptor.directory;
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstable);
-
-        transactionLogs.finish();
-        sstable.markObsolete(tidier);
-        sstable.selfRef().release();
-
-        for (int i = 0; i < 1000; i++)
-        {
-            // This should race with the asynchronous deletion of txn log files
-            // It doesn't matter what it returns but it should not throw
-            TransactionLogs.getTemporaryFiles(cfs.metadata, dataFolder);
-        }
-    }
-
-    private static SSTableReader sstable(ColumnFamilyStore cfs, int generation, int size) throws IOException
-    {
-        Directories dir = new Directories(cfs.metadata);
-        Descriptor descriptor = new Descriptor(dir.getDirectoryForNewSSTables(), cfs.keyspace.getName(), cfs.getColumnFamilyName(), generation);
-        Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
-        for (Component component : components)
-        {
-            File file = new File(descriptor.filenameFor(component));
-            file.createNewFile();
-            try (RandomAccessFile raf = new RandomAccessFile(file, "rw"))
-            {
-                raf.setLength(size);
-            }
-        }
-
-        SegmentedFile dFile = new BufferedSegmentedFile(new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))), RandomAccessReader.DEFAULT_BUFFER_SIZE, 0);
-        SegmentedFile iFile = new BufferedSegmentedFile(new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))), RandomAccessReader.DEFAULT_BUFFER_SIZE, 0);
-
-        SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.EMPTY_LIST);
-        StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata.comparator)
-                                                 .finalizeMetadata(cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
-                                                 .get(MetadataType.STATS);
-        SSTableReader reader = SSTableReader.internalOpen(descriptor,
-                                                          components,
-                                                          cfs.metadata,
-                                                          dFile,
-                                                          iFile,
-                                                          MockSchema.indexSummary.sharedCopy(),
-                                                          new AlwaysPresentFilter(),
-                                                          1L,
-                                                          metadata,
-                                                          SSTableReader.OpenReason.NORMAL,
-                                                          header);
-        reader.first = reader.last = MockSchema.readerBounds(generation);
-        return reader;
-    }
-
-    private static void assertFiles(String dirPath, Set<String> expectedFiles)
-    {
-        File dir = new File(dirPath);
-        for (File file : dir.listFiles())
-        {
-            if (file.isDirectory())
-                continue;
-
-            String filePath = file.getPath();
-            assertTrue(filePath, expectedFiles.contains(filePath));
-            expectedFiles.remove(filePath);
-        }
-
-        assertTrue(expectedFiles.isEmpty());
-    }
-}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
index 9a558f1..ceeb369 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
@@ -99,12 +99,5 @@ public class CQLSSTableWriterClientTest
 
         File[] dataFiles = this.testDirectory.listFiles(filter);
         assertEquals(2, dataFiles.length);
-
-        File transactionsFolder = Directories.getTransactionsDirectory(testDirectory);
-        assertTrue(transactionsFolder.exists());
-
-        File[] opFiles = transactionsFolder.listFiles();
-        assertEquals(0, opFiles.length);
-
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
index 2e9768e..e7cf51c 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
@@ -45,7 +45,6 @@ import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.OutputHandler;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 
 public class CQLSSTableWriterTest
 {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index fd801ad..d9516cb 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -44,7 +44,7 @@ import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
 import org.apache.cassandra.db.compaction.CompactionController;
@@ -109,7 +109,7 @@ public class SSTableRewriterTest extends SchemaLoader
         Keyspace keyspace = Keyspace.open(KEYSPACE);
         ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
         store.truncateBlocking();
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
     }
 
     @Test
@@ -145,7 +145,7 @@ public class SSTableRewriterTest extends SchemaLoader
             }
             writer.finish();
         }
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         validateCFS(cfs);
         int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
         assertEquals(1, filecounts);
@@ -177,7 +177,7 @@ public class SSTableRewriterTest extends SchemaLoader
             }
             writer.finish();
         }
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         validateCFS(cfs);
         int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
         assertEquals(1, filecounts);
@@ -232,7 +232,7 @@ public class SSTableRewriterTest extends SchemaLoader
             assertTrue(checked);
             writer.finish();
         }
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         validateCFS(cfs);
         int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
         assertEquals(1, filecounts);
@@ -277,12 +277,12 @@ public class SSTableRewriterTest extends SchemaLoader
             // open till .abort() is called (via the builder)
             if (!FBUtilities.isWindows())
             {
-                TransactionLogs.waitForDeletions();
+                TransactionLog.waitForDeletions();
                 assertFileCounts(dir.list());
             }
             writer.abort();
             txn.abort();
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             int datafiles = assertFileCounts(dir.list());
             assertEquals(datafiles, 0);
             validateCFS(cfs);
@@ -328,7 +328,7 @@ public class SSTableRewriterTest extends SchemaLoader
             sstables = rewriter.finish();
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         long sum = 0;
         for (SSTableReader x : cfs.getLiveSSTables())
@@ -337,7 +337,7 @@ public class SSTableRewriterTest extends SchemaLoader
         assertEquals(startStorageMetricsLoad - sBytesOnDisk + sum, StorageMetrics.load.getCount());
         assertEquals(files, sstables.size());
         assertEquals(files, cfs.getLiveSSTables().size());
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         // tmplink and tmp files should be gone:
         assertEquals(sum, cfs.metric.totalDiskSpaceUsed.getCount());
@@ -382,7 +382,7 @@ public class SSTableRewriterTest extends SchemaLoader
 
         assertEquals(files, sstables.size());
         assertEquals(files, cfs.getLiveSSTables().size());
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         assertFileCounts(s.descriptor.directory.list());
         validateCFS(cfs);
@@ -519,7 +519,7 @@ public class SSTableRewriterTest extends SchemaLoader
             test.run(scanner, controller, s, cfs, rewriter, txn);
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         assertEquals(startSize, cfs.metric.liveDiskSpaceUsed.getCount());
         assertEquals(1, cfs.getLiveSSTables().size());
@@ -567,7 +567,7 @@ public class SSTableRewriterTest extends SchemaLoader
             }
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         assertEquals(files - 1, cfs.getLiveSSTables().size()); // we never wrote anything to the last file
         assertFileCounts(s.descriptor.directory.list());
@@ -609,7 +609,7 @@ public class SSTableRewriterTest extends SchemaLoader
             sstables = rewriter.finish();
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         assertFileCounts(s.descriptor.directory.list());
         validateCFS(cfs);
     }
@@ -650,7 +650,7 @@ public class SSTableRewriterTest extends SchemaLoader
         }
         assertEquals(files, sstables.size());
         assertEquals(files, cfs.getLiveSSTables().size());
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         assertFileCounts(s.descriptor.directory.list());
 
         validateCFS(cfs);
@@ -670,7 +670,7 @@ public class SSTableRewriterTest extends SchemaLoader
             splitter.split();
 
             assertFileCounts(s.descriptor.directory.list());
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
 
             for (File f : s.descriptor.directory.listFiles())
             {
@@ -746,7 +746,7 @@ public class SSTableRewriterTest extends SchemaLoader
                 s.selfRef().release();
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         int filecount = assertFileCounts(s.descriptor.directory.list());
         assertEquals(filecount, 1);
@@ -825,7 +825,7 @@ public class SSTableRewriterTest extends SchemaLoader
             rewriter.finish();
         }
         validateKeys(keyspace);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         validateCFS(cfs);
         truncate(cfs);
     }
@@ -923,7 +923,7 @@ public class SSTableRewriterTest extends SchemaLoader
     public static void truncate(ColumnFamilyStore cfs)
     {
         cfs.truncateBlocking();
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS);
         assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
         assertEquals(0, cfs.metric.totalDiskSpaceUsed.getCount());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index c763932..0898515 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -51,17 +51,10 @@ public class MetadataSerializerTest
 
         CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
 
-
         ReplayPosition rp = new ReplayPosition(11L, 12);
 
-
         MetadataCollector collector = new MetadataCollector(cfm.comparator).replayPosition(rp);
 
-
-        Set<Integer> ancestors = Sets.newHashSet(1, 2, 3, 4);
-        for (int i : ancestors)
-            collector.addAncestor(i);
-
         String partitioner = RandomPartitioner.class.getCanonicalName();
         double bfFpChance = 0.1;
         Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.EMPTY_LIST));

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/schema/DefsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/schema/DefsTest.java b/test/unit/org/apache/cassandra/schema/DefsTest.java
index 680c016..98a954c 100644
--- a/test/unit/org/apache/cassandra/schema/DefsTest.java
+++ b/test/unit/org/apache/cassandra/schema/DefsTest.java
@@ -39,8 +39,9 @@ import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.ConfigurationException;
@@ -205,7 +206,7 @@ public class DefsTest
         ColumnFamilyStore store = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
         assertNotNull(store);
         store.forceBlockingFlush();
-        assertTrue(store.directories.sstableLister().list().size() > 0);
+        assertTrue(store.directories.sstableLister(Directories.OnTxnErr.THROW).list().size() > 0);
 
         MigrationManager.announceColumnFamilyDrop(ks.name, cfm.cfName);
 
@@ -227,7 +228,7 @@ public class DefsTest
 
         // verify that the files are gone.
         Supplier<Object> lambda = () -> {
-            for (File file : store.directories.sstableLister().listFiles())
+            for (File file : store.directories.sstableLister(Directories.OnTxnErr.THROW).listFiles())
             {
                 if (file.getPath().endsWith("Data.db") && !new File(file.getPath().replace("Data.db", "Compacted")).exists())
                     return false;
@@ -276,7 +277,7 @@ public class DefsTest
         ColumnFamilyStore cfs = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
         assertNotNull(cfs);
         cfs.forceBlockingFlush();
-        assertTrue(!cfs.directories.sstableLister().list().isEmpty());
+        assertTrue(!cfs.directories.sstableLister(Directories.OnTxnErr.THROW).list().isEmpty());
 
         MigrationManager.announceKeyspaceDrop(ks.name);
 
@@ -521,7 +522,7 @@ public class DefsTest
 
         // check
         assertTrue(cfs.indexManager.getIndexes().isEmpty());
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         assertFalse(new File(desc.filenameFor(Component.DATA)).exists());
     }
 


[4/7] cassandra git commit: Improve transaction log under FS corruption

Posted by be...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/DirectoriesTest.java b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
index 14db2d1..e0786f9 100644
--- a/test/unit/org/apache/cassandra/db/DirectoriesTest.java
+++ b/test/unit/org/apache/cassandra/db/DirectoriesTest.java
@@ -23,7 +23,6 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
-import com.google.common.collect.Lists;
 import org.apache.commons.lang3.StringUtils;
 
 import org.junit.AfterClass;
@@ -236,7 +235,7 @@ public class DirectoriesTest
             Set<File> listed;
 
             // List all but no snapshot, backup
-            lister = directories.sstableLister();
+            lister = directories.sstableLister(Directories.OnTxnErr.THROW);
             listed = new HashSet<>(lister.listFiles());
             for (File f : files.get(cfm.cfName))
             {
@@ -247,7 +246,7 @@ public class DirectoriesTest
             }
 
             // List all but including backup (but no snapshot)
-            lister = directories.sstableLister().includeBackups(true);
+            lister = directories.sstableLister(Directories.OnTxnErr.THROW).includeBackups(true);
             listed = new HashSet<>(lister.listFiles());
             for (File f : files.get(cfm.cfName))
             {
@@ -258,7 +257,7 @@ public class DirectoriesTest
             }
 
             // Skip temporary and compacted
-            lister = directories.sstableLister().skipTemporary(true);
+            lister = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
             listed = new HashSet<>(lister.listFiles());
             for (File f : files.get(cfm.cfName))
             {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/KeyCacheTest.java b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
index 30812d2..aedba0e 100644
--- a/test/unit/org/apache/cassandra/db/KeyCacheTest.java
+++ b/test/unit/org/apache/cassandra/db/KeyCacheTest.java
@@ -37,6 +37,7 @@ import org.apache.cassandra.config.CFMetaData;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.db.compaction.CompactionManager;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.schema.KeyspaceParams;
@@ -163,10 +164,7 @@ public class KeyCacheTest
 
         refs.release();
 
-        while (ScheduledExecutors.nonPeriodicTasks.getActiveCount() + ScheduledExecutors.nonPeriodicTasks.getQueue().size() > 0)
-        {
-            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);;
-        }
+        TransactionLog.waitForDeletions();
 
         // after releasing the reference this should drop to 2
         assertKeyCacheSize(2, KEYSPACE1, COLUMN_FAMILY1);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/ScrubTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ScrubTest.java b/test/unit/org/apache/cassandra/db/ScrubTest.java
index cc4038d..8889488 100644
--- a/test/unit/org/apache/cassandra/db/ScrubTest.java
+++ b/test/unit/org/apache/cassandra/db/ScrubTest.java
@@ -43,7 +43,7 @@ import org.apache.cassandra.db.compaction.CompactionManager;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.db.compaction.Scrubber;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.db.marshal.*;
 import org.apache.cassandra.db.marshal.UUIDType;
 import org.apache.cassandra.db.partitions.Partition;
@@ -371,7 +371,7 @@ public class ScrubTest
             {
                 scrubber.scrub();
             }
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             cfs.loadNewSSTables();
             assertOrderedAll(cfs, 7);
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
index e9c903e..7f1b2bd 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/HelpersTest.java
@@ -32,14 +32,9 @@ import org.junit.Test;
 
 import junit.framework.Assert;
 import org.apache.cassandra.MockSchema;
-import org.apache.cassandra.Util;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.format.big.BigTableReader;
-import org.apache.cassandra.utils.Pair;
-import org.apache.cassandra.utils.concurrent.Refs;
 
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertNotNull;
@@ -164,10 +159,10 @@ public class HelpersTest
     public void testMarkObsolete()
     {
         ColumnFamilyStore cfs = MockSchema.newCFS();
-        TransactionLogs txnLogs = new TransactionLogs(OperationType.UNKNOWN, cfs.metadata);
+        TransactionLog txnLogs = new TransactionLog(OperationType.UNKNOWN, cfs.metadata);
         Iterable<SSTableReader> readers = Lists.newArrayList(MockSchema.sstable(1, cfs), MockSchema.sstable(2, cfs));
 
-        List<TransactionLogs.Obsoletion> obsoletions = new ArrayList<>();
+        List<TransactionLog.Obsoletion> obsoletions = new ArrayList<>();
         Assert.assertNull(Helpers.prepareForObsoletion(readers, txnLogs, obsoletions, null));
         assertNotNull(obsoletions);
         assertEquals(2, obsoletions.size());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
index a376a61..db27662 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/LifecycleTransactionTest.java
@@ -249,7 +249,7 @@ public class LifecycleTransactionTest extends AbstractTransactionalTest
 
     protected TestableTransaction newTest()
     {
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         SSTableReader.resetTidying();
         return new TxnTest();
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
index 44f4d30..309e35a 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/RealTransactionsTest.java
@@ -89,10 +89,9 @@ public class RealTransactionsTest extends SchemaLoader
         SSTableReader oldSSTable = getSSTable(cfs, 1);
         LifecycleTransaction txn = cfs.getTracker().tryModify(oldSSTable, OperationType.COMPACTION);
         SSTableReader newSSTable = replaceSSTable(cfs, txn, false);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
-        assertFiles(txn.logs().getDataFolder(), new HashSet<>(newSSTable.getAllFilePaths()));
-        assertFiles(txn.logs().getLogsFolder(), Collections.<String>emptySet());
+        assertFiles(txn.log().getDataFolder(), new HashSet<>(newSSTable.getAllFilePaths()));
     }
 
     @Test
@@ -105,10 +104,9 @@ public class RealTransactionsTest extends SchemaLoader
         LifecycleTransaction txn = cfs.getTracker().tryModify(oldSSTable, OperationType.COMPACTION);
 
         replaceSSTable(cfs, txn, true);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
-        assertFiles(txn.logs().getDataFolder(), new HashSet<>(oldSSTable.getAllFilePaths()));
-        assertFiles(txn.logs().getLogsFolder(), Collections.<String>emptySet());
+        assertFiles(txn.log().getDataFolder(), new HashSet<>(oldSSTable.getAllFilePaths()));
     }
 
     @Test
@@ -120,11 +118,6 @@ public class RealTransactionsTest extends SchemaLoader
         SSTableReader ssTableReader = getSSTable(cfs, 100);
 
         String dataFolder = cfs.getLiveSSTables().iterator().next().descriptor.directory.getPath();
-        String transactionLogsFolder = StringUtils.join(dataFolder, File.separator, Directories.TRANSACTIONS_SUBDIR);
-
-        assertTrue(new File(transactionLogsFolder).exists());
-        assertFiles(transactionLogsFolder, Collections.<String>emptySet());
-
         assertFiles(dataFolder, new HashSet<>(ssTableReader.getAllFilePaths()));
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
index ea0d9a8..3a943c4 100644
--- a/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TrackerTest.java
@@ -187,9 +187,9 @@ public class TrackerTest
     public void testDropSSTables()
     {
         testDropSSTables(false);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         testDropSSTables(true);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
     }
 
     private void testDropSSTables(boolean invalidate)
@@ -203,62 +203,54 @@ public class TrackerTest
                                                              MockSchema.sstable(2, 71, true, cfs));
         tracker.addInitialSSTables(copyOf(readers));
 
-        try
+        try (LifecycleTransaction txn = tracker.tryModify(readers.get(0), OperationType.COMPACTION))
         {
-           // TransactionLogs.pauseDeletions(true);
-            try (LifecycleTransaction txn = tracker.tryModify(readers.get(0), OperationType.COMPACTION))
+            if (invalidate)
             {
-                if (invalidate)
-                {
-                    cfs.invalidate(false);
-                }
-                else
-                {
-                    tracker.dropSSTables();
-                    TransactionLogs.waitForDeletions();
-                }
-                Assert.assertEquals(9, cfs.metric.totalDiskSpaceUsed.getCount());
-                Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
-                Assert.assertEquals(1, tracker.getView().sstables.size());
+                cfs.invalidate(false);
             }
-            if (!invalidate)
+            else
             {
-                Assert.assertEquals(1, tracker.getView().sstables.size());
-                Assert.assertEquals(readers.get(0), Iterables.getFirst(tracker.getView().sstables, null));
-                Assert.assertEquals(1, readers.get(0).selfRef().globalCount());
-                Assert.assertFalse(readers.get(0).isMarkedCompacted());
-                for (SSTableReader reader : readers.subList(1, 3))
-                {
-                    Assert.assertEquals(0, reader.selfRef().globalCount());
-                    Assert.assertTrue(reader.isMarkedCompacted());
-                }
-
-                Assert.assertNull(tracker.dropSSTables(reader -> reader != readers.get(0), OperationType.UNKNOWN, null));
-
-                Assert.assertEquals(1, tracker.getView().sstables.size());
-                Assert.assertEquals(3, listener.received.size());
-                Assert.assertEquals(tracker, listener.senders.get(0));
-                Assert.assertTrue(listener.received.get(0) instanceof SSTableDeletingNotification);
-                Assert.assertTrue(listener.received.get(1) instanceof  SSTableDeletingNotification);
-                Assert.assertTrue(listener.received.get(2) instanceof SSTableListChangedNotification);
-                Assert.assertEquals(readers.get(1), ((SSTableDeletingNotification) listener.received.get(0)).deleting);
-                Assert.assertEquals(readers.get(2), ((SSTableDeletingNotification)listener.received.get(1)).deleting);
-                Assert.assertEquals(2, ((SSTableListChangedNotification) listener.received.get(2)).removed.size());
-                Assert.assertEquals(0, ((SSTableListChangedNotification) listener.received.get(2)).added.size());
-                Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
-                readers.get(0).selfRef().release();
+                tracker.dropSSTables();
+                TransactionLog.waitForDeletions();
             }
-            else
+            Assert.assertEquals(9, cfs.metric.totalDiskSpaceUsed.getCount());
+            Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
+            Assert.assertEquals(1, tracker.getView().sstables.size());
+        }
+        if (!invalidate)
+        {
+            Assert.assertEquals(1, tracker.getView().sstables.size());
+            Assert.assertEquals(readers.get(0), Iterables.getFirst(tracker.getView().sstables, null));
+            Assert.assertEquals(1, readers.get(0).selfRef().globalCount());
+            Assert.assertFalse(readers.get(0).isMarkedCompacted());
+            for (SSTableReader reader : readers.subList(1, 3))
             {
-                Assert.assertEquals(0, tracker.getView().sstables.size());
-                Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
-                for (SSTableReader reader : readers)
-                    Assert.assertTrue(reader.isMarkedCompacted());
+                Assert.assertEquals(0, reader.selfRef().globalCount());
+                Assert.assertTrue(reader.isMarkedCompacted());
             }
+
+            Assert.assertNull(tracker.dropSSTables(reader -> reader != readers.get(0), OperationType.UNKNOWN, null));
+
+            Assert.assertEquals(1, tracker.getView().sstables.size());
+            Assert.assertEquals(3, listener.received.size());
+            Assert.assertEquals(tracker, listener.senders.get(0));
+            Assert.assertTrue(listener.received.get(0) instanceof SSTableDeletingNotification);
+            Assert.assertTrue(listener.received.get(1) instanceof  SSTableDeletingNotification);
+            Assert.assertTrue(listener.received.get(2) instanceof SSTableListChangedNotification);
+            Assert.assertEquals(readers.get(1), ((SSTableDeletingNotification) listener.received.get(0)).deleting);
+            Assert.assertEquals(readers.get(2), ((SSTableDeletingNotification)listener.received.get(1)).deleting);
+            Assert.assertEquals(2, ((SSTableListChangedNotification) listener.received.get(2)).removed.size());
+            Assert.assertEquals(0, ((SSTableListChangedNotification) listener.received.get(2)).added.size());
+            Assert.assertEquals(9, cfs.metric.liveDiskSpaceUsed.getCount());
+            readers.get(0).selfRef().release();
         }
-        finally
+        else
         {
-           // TransactionLogs.pauseDeletions(false);
+            Assert.assertEquals(0, tracker.getView().sstables.size());
+            Assert.assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
+            for (SSTableReader reader : readers)
+                Assert.assertTrue(reader.isMarkedCompacted());
         }
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogTest.java
new file mode 100644
index 0000000..7739163
--- /dev/null
+++ b/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogTest.java
@@ -0,0 +1,791 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import junit.framework.Assert;
+import org.apache.cassandra.MockSchema;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.SerializationHeader;
+import org.apache.cassandra.db.compaction.*;
+import org.apache.cassandra.io.sstable.*;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
+import org.apache.cassandra.io.sstable.metadata.MetadataType;
+import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
+import org.apache.cassandra.io.util.BufferedSegmentedFile;
+import org.apache.cassandra.io.util.ChannelProxy;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.io.util.RandomAccessReader;
+import org.apache.cassandra.io.util.SegmentedFile;
+import org.apache.cassandra.utils.AlwaysPresentFilter;
+import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+public class TransactionLogTest extends AbstractTransactionalTest
+{
+    private static final String KEYSPACE = "TransactionLogsTest";
+
+    @BeforeClass
+    public static void setUp()
+    {
+        MockSchema.cleanup();
+    }
+
+    protected AbstractTransactionalTest.TestableTransaction newTest() throws Exception
+    {
+        TransactionLog.waitForDeletions();
+        SSTableReader.resetTidying();
+        return new TxnTest();
+    }
+
+    private static final class TxnTest extends TestableTransaction
+    {
+        private final static class Transaction extends Transactional.AbstractTransactional implements Transactional
+        {
+            final ColumnFamilyStore cfs;
+            final TransactionLog txnLogs;
+            final SSTableReader sstableOld;
+            final SSTableReader sstableNew;
+            final TransactionLog.SSTableTidier tidier;
+
+            public Transaction(ColumnFamilyStore cfs, TransactionLog txnLogs) throws IOException
+            {
+                this.cfs = cfs;
+                this.txnLogs = txnLogs;
+                this.sstableOld = sstable(cfs, 0, 128);
+                this.sstableNew = sstable(cfs, 1, 128);
+
+                assertNotNull(txnLogs);
+                assertNotNull(txnLogs.getId());
+                Assert.assertEquals(OperationType.COMPACTION, txnLogs.getType());
+
+                txnLogs.trackNew(sstableNew);
+                tidier = txnLogs.obsoleted(sstableOld);
+                assertNotNull(tidier);
+            }
+
+            protected Throwable doCommit(Throwable accumulate)
+            {
+                sstableOld.markObsolete(tidier);
+                sstableOld.selfRef().release();
+                TransactionLog.waitForDeletions();
+
+                Throwable ret = txnLogs.commit(accumulate);
+
+                sstableNew.selfRef().release();
+                return ret;
+            }
+
+            protected Throwable doAbort(Throwable accumulate)
+            {
+                tidier.abort();
+                TransactionLog.waitForDeletions();
+
+                Throwable ret = txnLogs.abort(accumulate);
+
+                sstableNew.selfRef().release();
+                sstableOld.selfRef().release();
+                return ret;
+            }
+
+            protected void doPrepare()
+            {
+                txnLogs.prepareToCommit();
+            }
+
+            protected void assertInProgress() throws Exception
+            {
+                assertFiles(txnLogs.getDataFolder(), Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths(),
+                                                                                      sstableOld.getAllFilePaths(),
+                                                                                      Collections.singleton(txnLogs.getData().getLogFile().file.getPath()))));
+            }
+
+            protected void assertPrepared() throws Exception
+            {
+            }
+
+            protected void assertAborted() throws Exception
+            {
+                assertFiles(txnLogs.getDataFolder(), new HashSet<>(sstableOld.getAllFilePaths()));
+            }
+
+            protected void assertCommitted() throws Exception
+            {
+                assertFiles(txnLogs.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
+            }
+        }
+
+        final Transaction txn;
+
+        private TxnTest() throws IOException
+        {
+            this(MockSchema.newCFS(KEYSPACE));
+        }
+
+        private TxnTest(ColumnFamilyStore cfs) throws IOException
+        {
+            this(cfs, new TransactionLog(OperationType.COMPACTION, cfs.metadata));
+        }
+
+        private TxnTest(ColumnFamilyStore cfs, TransactionLog txnLogs) throws IOException
+        {
+            this(new Transaction(cfs, txnLogs));
+        }
+
+        private TxnTest(Transaction txn)
+        {
+            super(txn);
+            this.txn = txn;
+        }
+
+        protected void assertInProgress() throws Exception
+        {
+            txn.assertInProgress();
+        }
+
+        protected void assertPrepared() throws Exception
+        {
+            txn.assertPrepared();
+        }
+
+        protected void assertAborted() throws Exception
+        {
+            txn.assertAborted();
+        }
+
+        protected void assertCommitted() throws Exception
+        {
+            txn.assertCommitted();
+        }
+    }
+
+    @Test
+    public void testUntrack() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        // complete a transaction without keep the new files since they were untracked
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        transactionLog.untrackNew(sstableNew);
+
+        transactionLog.finish();
+
+        sstableNew.selfRef().release();
+        Thread.sleep(1);
+        TransactionLog.waitForDeletions();
+
+        assertFiles(transactionLog.getDataFolder(), Collections.<String>emptySet());
+    }
+
+    @Test
+    public void testCommitSameDesc() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld1 = sstable(cfs, 0, 128);
+        SSTableReader sstableOld2 = sstable(cfs, 0, 256);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+
+        sstableOld1.setReplaced();
+
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstableOld2);
+        assertNotNull(tidier);
+
+        transactionLog.finish();
+
+        sstableOld2.markObsolete(tidier);
+
+        sstableOld1.selfRef().release();
+        sstableOld2.selfRef().release();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
+
+        sstableNew.selfRef().release();
+    }
+
+    @Test
+    public void testCommitOnlyNew() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstable);
+        transactionLog.finish();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstable.getAllFilePaths()));
+
+        sstable.selfRef().release();
+    }
+
+    @Test
+    public void testCommitOnlyOld() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstable);
+        assertNotNull(tidier);
+
+        transactionLog.finish();
+        sstable.markObsolete(tidier);
+        sstable.selfRef().release();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>());
+    }
+
+    @Test
+    public void testAbortOnlyNew() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstable);
+        transactionLog.abort();
+
+        sstable.selfRef().release();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>());
+    }
+
+    @Test
+    public void testAbortOnlyOld() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstable);
+        assertNotNull(tidier);
+
+        tidier.abort();
+        transactionLog.abort();
+
+        sstable.selfRef().release();
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstable.getAllFilePaths()));
+    }
+
+    @Test
+    public void testRemoveUnfinishedLeftovers_abort() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld = sstable(cfs, 0, 128);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        // simulate tracking sstables with a failed transaction (new log file NOT deleted)
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstableOld);
+
+        Set<File> tmpFiles = Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths().stream().map(p -> new File(p)).collect(Collectors.toList()),
+                                                              Collections.singleton(transactionLog.getData().getLogFile().file)));
+
+        sstableNew.selfRef().release();
+        sstableOld.selfRef().release();
+
+        Assert.assertEquals(tmpFiles, TransactionLog.getTemporaryFiles(cfs.metadata, sstableNew.descriptor.directory));
+
+        // normally called at startup
+        TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+        // sstableOld should be only table left
+        Directories directories = new Directories(cfs.metadata);
+        Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
+        assertEquals(1, sstables.size());
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstableOld.getAllFilePaths()));
+
+        tidier.run();
+
+        // complete the transaction to avoid LEAK errors
+        transactionLog.close();
+    }
+
+    @Test
+    public void testRemoveUnfinishedLeftovers_commit() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld = sstable(cfs, 0, 128);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        // simulate tracking sstables with a committed transaction (new log file deleted)
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        TransactionLog.SSTableTidier tidier = transactionLog.obsoleted(sstableOld);
+
+        //Fake a commit
+        transactionLog.getData().getLogFile().commit();
+
+        Set<File> tmpFiles = Sets.newHashSet(Iterables.concat(sstableOld.getAllFilePaths().stream().map(p -> new File(p)).collect(Collectors.toList()),
+                                                              Collections.singleton(transactionLog.getData().getLogFile().file)));
+
+        sstableNew.selfRef().release();
+        sstableOld.selfRef().release();
+
+        Assert.assertEquals(tmpFiles, TransactionLog.getTemporaryFiles(cfs.metadata, sstableOld.descriptor.directory));
+
+        // normally called at startup
+        TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+        // sstableNew should be only table left
+        Directories directories = new Directories(cfs.metadata);
+        Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
+        assertEquals(1, sstables.size());
+
+        assertFiles(transactionLog.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
+
+        tidier.run();
+
+        // complete the transaction to avoid LEAK errors
+        assertNull(transactionLog.complete(null));
+    }
+
+    @Test
+    public void testGetTemporaryFiles() throws IOException
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable1 = sstable(cfs, 0, 128);
+
+        File dataFolder = sstable1.descriptor.directory;
+
+        Set<File> tmpFiles = TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder);
+        assertNotNull(tmpFiles);
+        assertEquals(0, tmpFiles.size());
+
+        TransactionLog transactionLog = new TransactionLog(OperationType.WRITE, cfs.metadata);
+        Directories directories = new Directories(cfs.metadata);
+
+        File[] beforeSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
+
+        SSTableReader sstable2 = sstable(cfs, 1, 128);
+        transactionLog.trackNew(sstable2);
+
+        Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
+        assertEquals(2, sstables.size());
+
+        // this should contain sstable1, sstable2 and the transaction log file
+        File[] afterSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
+
+        int numNewFiles = afterSecondSSTable.length - beforeSecondSSTable.length;
+        assertEquals(numNewFiles - 1, sstable2.getAllFilePaths().size()); // new files except for transaction log file
+
+        tmpFiles = TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder);
+        assertNotNull(tmpFiles);
+        assertEquals(numNewFiles, tmpFiles.size());
+
+        File ssTable2DataFile = new File(sstable2.descriptor.filenameFor(Component.DATA));
+        File ssTable2IndexFile = new File(sstable2.descriptor.filenameFor(Component.PRIMARY_INDEX));
+
+        assertTrue(tmpFiles.contains(ssTable2DataFile));
+        assertTrue(tmpFiles.contains(ssTable2IndexFile));
+
+        List<File> files = directories.sstableLister(Directories.OnTxnErr.THROW).listFiles();
+        List<File> filesNoTmp = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
+        assertNotNull(files);
+        assertNotNull(filesNoTmp);
+
+        assertTrue(files.contains(ssTable2DataFile));
+        assertTrue(files.contains(ssTable2IndexFile));
+
+        assertFalse(filesNoTmp.contains(ssTable2DataFile));
+        assertFalse(filesNoTmp.contains(ssTable2IndexFile));
+
+        transactionLog.finish();
+
+        //Now it should be empty since the transaction has finished
+        tmpFiles = TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder);
+        assertNotNull(tmpFiles);
+        assertEquals(0, tmpFiles.size());
+
+        filesNoTmp = directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true).listFiles();
+        assertNotNull(filesNoTmp);
+        assertTrue(filesNoTmp.contains(ssTable2DataFile));
+        assertTrue(filesNoTmp.contains(ssTable2IndexFile));
+
+        sstable1.selfRef().release();
+        sstable2.selfRef().release();
+    }
+
+    @Test
+    public void testWrongChecksumLastLine() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake a commit with invalid checksum
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+                          },
+                          true);
+    }
+
+    @Test
+    public void testWrongChecksumSecondFromLastLine() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake two lines with invalid checksum
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("add:[ma-3-big,%d,4][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+                          },
+                          false);
+    }
+
+    @Test
+    public void testWrongChecksumLastLineMissingFile() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake a commit with invalid checksum and also delete one of the old files
+                              for (String filePath : s.getAllFilePaths())
+                              {
+                                  if (filePath.endsWith("Data.db"))
+                                  {
+                                      FileUtils.delete(filePath);
+                                      break;
+                                  }
+                              }
+
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+                          },
+                          false);
+    }
+
+    @Test
+    public void testWrongChecksumLastLineWrongRecordFormat() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake a commit with invalid checksum and a wrong record format (extra spaces)
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d ,0 ,0 ][%d]",
+                                                             System.currentTimeMillis(),
+                                                             12345678L));
+                          },
+                          true);
+    }
+
+    @Test
+    public void testMissingChecksumLastLine() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          {
+                              // Fake a commit without a checksum
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0]",
+                                                             System.currentTimeMillis()));
+                          },
+                          true);
+    }
+
+    @Test
+    public void testMissingChecksumSecondFromLastLine() throws IOException
+    {
+        testCorruptRecord((t, s) ->
+                          { // Fake two lines without a checksum
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("add:[ma-3-big,%d,4]",
+                                                             System.currentTimeMillis()));
+
+                              FileUtils.append(t.getData().getLogFile().file,
+                                               String.format("commit:[%d,0,0]",
+                                                             System.currentTimeMillis()));
+                          },
+                          false);
+    }
+
+    private void testCorruptRecord(BiConsumer<TransactionLog, SSTableReader> modifier, boolean isRecoverable) throws IOException
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld = sstable(cfs, 0, 128);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        File dataFolder = sstableOld.descriptor.directory;
+
+        // simulate tracking sstables with a committed transaction except the checksum will be wrong
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        transactionLog.obsoleted(sstableOld);
+
+        //Modify the transaction log in some way
+        modifier.accept(transactionLog, sstableOld);
+
+        String txnFilePath = transactionLog.getData().getLogFile().file.getPath();
+
+        transactionLog.complete(null);
+
+        sstableOld.selfRef().release();
+        sstableNew.selfRef().release();
+
+        if (isRecoverable)
+        { // the corruption is recoverable, we assume there is a commit record
+
+            //This should return the old files and the tx log
+            assertFiles(Iterables.concat(sstableOld.getAllFilePaths(), Collections.singleton(txnFilePath)),
+                        TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder));
+
+            //This should remove old files
+            TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+            assertFiles(dataFolder.getPath(), Sets.newHashSet(sstableNew.getAllFilePaths()));
+        }
+        else
+        { // if an intermediate line was modified, we cannot tell,
+          // it should just throw and handle the exception with a log message
+
+            //This should not return any files
+            assertEquals(Collections.emptyList(), new TransactionLog.FileLister(dataFolder.toPath(),
+                                                                                (file, type) -> type != Directories.FileType.FINAL,
+                                                                                Directories.OnTxnErr.IGNORE).list());
+
+            try
+            {
+                //This should throw a RuntimeException
+                new TransactionLog.FileLister(dataFolder.toPath(),
+                                              (file, type) -> type != Directories.FileType.FINAL,
+                                              Directories.OnTxnErr.THROW).list();
+                fail("Expected exception");
+            }
+            catch (RuntimeException ex)
+            {
+                // pass
+                ex.printStackTrace();
+            }
+
+            //This should not remove any files
+            TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+            assertFiles(dataFolder.getPath(), Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths(),
+                                                                               sstableOld.getAllFilePaths(),
+                                                                               Collections.singleton(txnFilePath))),
+                        true);
+        }
+    }
+
+    @Test
+    public void testObsoletedDataFileUpdateTimeChanged() throws IOException
+    {
+        testObsoletedFilesChanged(sstable ->
+                                 {
+                                     // increase the modification time of the Data file
+                                     for (String filePath : sstable.getAllFilePaths())
+                                     {
+                                         if (filePath.endsWith("Data.db"))
+                                             assertTrue(new File(filePath).setLastModified(System.currentTimeMillis() + 60000)); //one minute later
+                                     }
+                                 });
+    }
+
+    private void testObsoletedFilesChanged(Consumer<SSTableReader> modifier) throws IOException
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstableOld = sstable(cfs, 0, 128);
+        SSTableReader sstableNew = sstable(cfs, 1, 128);
+
+        // simulate tracking sstables with a committed transaction except the checksum will be wrong
+        TransactionLog transactionLog = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLog);
+
+        transactionLog.trackNew(sstableNew);
+        /*TransactionLog.SSTableTidier tidier =*/ transactionLog.obsoleted(sstableOld);
+
+        //modify the old sstable files
+        modifier.accept(sstableOld);
+
+        //Fake a commit
+        transactionLog.getData().getLogFile().commit();
+
+        //This should not remove the old files
+        TransactionLog.removeUnfinishedLeftovers(cfs.metadata);
+
+        assertFiles(transactionLog.getDataFolder(), Sets.newHashSet(Iterables.concat(
+                                                                                    sstableNew.getAllFilePaths(),
+                                                                                    sstableOld.getAllFilePaths(),
+                                                                                    Collections.singleton(transactionLog.getData().getLogFile().file.getPath()))));
+
+        sstableOld.selfRef().release();
+        sstableNew.selfRef().release();
+
+        // complete the transaction to avoid LEAK errors
+        assertNull(transactionLog.complete(null));
+
+        assertFiles(transactionLog.getDataFolder(), Sets.newHashSet(Iterables.concat(
+                                                                                    sstableNew.getAllFilePaths(),
+                                                                                    sstableOld.getAllFilePaths(),
+                                                                                    Collections.singleton(transactionLog.getData().getLogFile().file.getPath()))));
+    }
+
+    @Test
+    public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable
+    {
+        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
+        SSTableReader sstable = sstable(cfs, 0, 128);
+        File dataFolder = sstable.descriptor.directory;
+
+        TransactionLog transactionLogs = new TransactionLog(OperationType.COMPACTION, cfs.metadata);
+        assertNotNull(transactionLogs);
+
+        TransactionLog.SSTableTidier tidier = transactionLogs.obsoleted(sstable);
+
+        transactionLogs.finish();
+        sstable.markObsolete(tidier);
+        sstable.selfRef().release();
+
+        for (int i = 0; i < 1000; i++)
+        {
+            // This should race with the asynchronous deletion of txn log files
+            // It doesn't matter what it returns but it should not throw
+            TransactionLog.getTemporaryFiles(cfs.metadata, dataFolder);
+        }
+    }
+
+    private static SSTableReader sstable(ColumnFamilyStore cfs, int generation, int size) throws IOException
+    {
+        Directories dir = new Directories(cfs.metadata);
+        Descriptor descriptor = new Descriptor(dir.getDirectoryForNewSSTables(), cfs.keyspace.getName(), cfs.getColumnFamilyName(), generation);
+        Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
+        for (Component component : components)
+        {
+            File file = new File(descriptor.filenameFor(component));
+            file.createNewFile();
+            try (RandomAccessFile raf = new RandomAccessFile(file, "rw"))
+            {
+                raf.setLength(size);
+            }
+        }
+
+        SegmentedFile dFile = new BufferedSegmentedFile(new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))), RandomAccessReader.DEFAULT_BUFFER_SIZE, 0);
+        SegmentedFile iFile = new BufferedSegmentedFile(new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))), RandomAccessReader.DEFAULT_BUFFER_SIZE, 0);
+
+        SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.EMPTY_LIST);
+        StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata.comparator)
+                                                 .finalizeMetadata(cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
+                                                 .get(MetadataType.STATS);
+        SSTableReader reader = SSTableReader.internalOpen(descriptor,
+                                                          components,
+                                                          cfs.metadata,
+                                                          dFile,
+                                                          iFile,
+                                                          MockSchema.indexSummary.sharedCopy(),
+                                                          new AlwaysPresentFilter(),
+                                                          1L,
+                                                          metadata,
+                                                          SSTableReader.OpenReason.NORMAL,
+                                                          header);
+        reader.first = reader.last = MockSchema.readerBounds(generation);
+        return reader;
+    }
+
+    private static void assertFiles(String dirPath, Set<String> expectedFiles)
+    {
+        assertFiles(dirPath, expectedFiles, false);
+    }
+
+    private static void assertFiles(String dirPath, Set<String> expectedFiles, boolean excludeNonExistingFiles)
+    {
+        TransactionLog.waitForDeletions();
+
+        File dir = new File(dirPath);
+        for (File file : dir.listFiles())
+        {
+            if (file.isDirectory())
+                continue;
+
+            String filePath = file.getPath();
+            assertTrue(filePath, expectedFiles.contains(filePath));
+            expectedFiles.remove(filePath);
+        }
+
+        if (excludeNonExistingFiles)
+        {
+            for (String filePath : expectedFiles)
+            {
+                File file = new File(filePath);
+                if (!file.exists())
+                    expectedFiles.remove(filePath);
+            }
+        }
+
+        assertTrue(expectedFiles.toString(), expectedFiles.isEmpty());
+    }
+
+    private static void assertFiles(Iterable<String> filePaths, Set<File> expectedFiles)
+    {
+        for (String filePath : filePaths)
+        {
+            File file = new File(filePath);
+            assertTrue(filePath, expectedFiles.contains(file));
+            expectedFiles.remove(file);
+        }
+
+        assertTrue(expectedFiles.isEmpty());
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogsTest.java b/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogsTest.java
deleted file mode 100644
index 991eed3..0000000
--- a/test/unit/org/apache/cassandra/db/lifecycle/TransactionLogsTest.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.lifecycle;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
-import java.util.*;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertNotNull;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import junit.framework.Assert;
-import org.apache.cassandra.MockSchema;
-import org.apache.cassandra.db.ColumnFamilyStore;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SerializationHeader;
-import org.apache.cassandra.db.compaction.*;
-import org.apache.cassandra.io.sstable.*;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.sstable.metadata.MetadataCollector;
-import org.apache.cassandra.io.sstable.metadata.MetadataType;
-import org.apache.cassandra.io.sstable.metadata.StatsMetadata;
-import org.apache.cassandra.io.util.BufferedSegmentedFile;
-import org.apache.cassandra.io.util.ChannelProxy;
-import org.apache.cassandra.io.util.RandomAccessReader;
-import org.apache.cassandra.io.util.SegmentedFile;
-import org.apache.cassandra.utils.AlwaysPresentFilter;
-import org.apache.cassandra.utils.concurrent.AbstractTransactionalTest;
-import org.apache.cassandra.utils.concurrent.Transactional;
-
-public class TransactionLogsTest extends AbstractTransactionalTest
-{
-    private static final String KEYSPACE = "TransactionLogsTest";
-
-    @BeforeClass
-    public static void setUp()
-    {
-        MockSchema.cleanup();
-    }
-
-    protected AbstractTransactionalTest.TestableTransaction newTest() throws Exception
-    {
-        TransactionLogs.waitForDeletions();
-        SSTableReader.resetTidying();
-        return new TxnTest();
-    }
-
-    private static final class TxnTest extends TestableTransaction
-    {
-        private final static class Transaction extends Transactional.AbstractTransactional implements Transactional
-        {
-            final ColumnFamilyStore cfs;
-            final TransactionLogs txnLogs;
-            final SSTableReader sstableOld;
-            final SSTableReader sstableNew;
-            final TransactionLogs.SSTableTidier tidier;
-
-            public Transaction(ColumnFamilyStore cfs, TransactionLogs txnLogs) throws IOException
-            {
-                this.cfs = cfs;
-                this.txnLogs = txnLogs;
-                this.sstableOld = sstable(cfs, 0, 128);
-                this.sstableNew = sstable(cfs, 1, 128);
-
-                assertNotNull(txnLogs);
-                assertNotNull(txnLogs.getId());
-                Assert.assertEquals(OperationType.COMPACTION, txnLogs.getType());
-
-                txnLogs.trackNew(sstableNew);
-                tidier = txnLogs.obsoleted(sstableOld);
-                assertNotNull(tidier);
-            }
-
-            protected Throwable doCommit(Throwable accumulate)
-            {
-                sstableOld.markObsolete(tidier);
-                sstableOld.selfRef().release();
-                TransactionLogs.waitForDeletions();
-
-                Throwable ret = txnLogs.commit(accumulate);
-
-                sstableNew.selfRef().release();
-                return ret;
-            }
-
-            protected Throwable doAbort(Throwable accumulate)
-            {
-                tidier.abort();
-                TransactionLogs.waitForDeletions();
-
-                Throwable ret = txnLogs.abort(accumulate);
-
-                sstableNew.selfRef().release();
-                sstableOld.selfRef().release();
-                return ret;
-            }
-
-            protected void doPrepare()
-            {
-                txnLogs.prepareToCommit();
-            }
-
-            protected void assertInProgress() throws Exception
-            {
-                assertFiles(txnLogs.getDataFolder(), Sets.newHashSet(Iterables.concat(sstableNew.getAllFilePaths(),
-                                                                                      sstableOld.getAllFilePaths())));
-                assertFiles(txnLogs.getLogsFolder(), Sets.newHashSet(txnLogs.getData().oldLog().file.getPath(),
-                                                                     txnLogs.getData().newLog().file.getPath()));
-                assertEquals(2, TransactionLogs.getLogFiles(cfs.metadata).size());
-            }
-
-            protected void assertPrepared() throws Exception
-            {
-            }
-
-            protected void assertAborted() throws Exception
-            {
-                assertFiles(txnLogs.getDataFolder(), new HashSet<>(sstableOld.getAllFilePaths()));
-                assertFiles(txnLogs.getLogsFolder(), Collections.<String>emptySet());
-                assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-            }
-
-            protected void assertCommitted() throws Exception
-            {
-                assertFiles(txnLogs.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
-                assertFiles(txnLogs.getLogsFolder(), Collections.<String>emptySet());
-                assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-            }
-        }
-
-        final Transaction txn;
-
-        private TxnTest() throws IOException
-        {
-            this(MockSchema.newCFS(KEYSPACE));
-        }
-
-        private TxnTest(ColumnFamilyStore cfs) throws IOException
-        {
-            this(cfs, new TransactionLogs(OperationType.COMPACTION, cfs.metadata));
-        }
-
-        private TxnTest(ColumnFamilyStore cfs, TransactionLogs txnLogs) throws IOException
-        {
-            this(new Transaction(cfs, txnLogs));
-        }
-
-        private TxnTest(Transaction txn)
-        {
-            super(txn);
-            this.txn = txn;
-        }
-
-        protected void assertInProgress() throws Exception
-        {
-            txn.assertInProgress();
-        }
-
-        protected void assertPrepared() throws Exception
-        {
-            txn.assertPrepared();
-        }
-
-        protected void assertAborted() throws Exception
-        {
-            txn.assertAborted();
-        }
-
-        protected void assertCommitted() throws Exception
-        {
-            txn.assertCommitted();
-        }
-    }
-
-    @Test
-    public void testUntrack() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstableNew = sstable(cfs, 1, 128);
-
-        // complete a transaction without keep the new files since they were untracked
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstableNew);
-        transactionLogs.untrackNew(sstableNew);
-
-        transactionLogs.finish();
-
-        assertFiles(transactionLogs.getDataFolder(), Collections.<String>emptySet());
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-
-        sstableNew.selfRef().release();
-    }
-
-    @Test
-    public void testCommitSameDesc() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstableOld1 = sstable(cfs, 0, 128);
-        SSTableReader sstableOld2 = sstable(cfs, 0, 256);
-        SSTableReader sstableNew = sstable(cfs, 1, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstableNew);
-
-        sstableOld1.setReplaced();
-
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstableOld2);
-        assertNotNull(tidier);
-
-        transactionLogs.finish();
-
-        sstableOld2.markObsolete(tidier);
-
-        sstableOld1.selfRef().release();
-        sstableOld2.selfRef().release();
-
-        TransactionLogs.waitForDeletions();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-
-        sstableNew.selfRef().release();
-    }
-
-    @Test
-    public void testCommitOnlyNew() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstable);
-        transactionLogs.finish();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstable.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-
-        sstable.selfRef().release();
-    }
-
-    @Test
-    public void testCommitOnlyOld() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstable);
-        assertNotNull(tidier);
-
-        transactionLogs.finish();
-        sstable.markObsolete(tidier);
-        sstable.selfRef().release();
-
-        TransactionLogs.waitForDeletions();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>());
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-    }
-
-    @Test
-    public void testAbortOnlyNew() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstable);
-        transactionLogs.abort();
-
-        sstable.selfRef().release();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>());
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-    }
-
-    @Test
-    public void testAbortOnlyOld() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstable);
-        assertNotNull(tidier);
-
-        tidier.abort();
-        transactionLogs.abort();
-
-        sstable.selfRef().release();
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstable.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-        assertEquals(0, TransactionLogs.getLogFiles(cfs.metadata).size());
-    }
-
-    private File copyToTmpFile(File file) throws IOException
-    {
-        File ret = File.createTempFile(file.getName(), ".tmp");
-        ret.deleteOnExit();
-        Files.copy(file.toPath(), ret.toPath(), StandardCopyOption.REPLACE_EXISTING);
-        return ret;
-    }
-
-    @Test
-    public void testRemoveUnfinishedLeftovers_newLogFound() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstableOld = sstable(cfs, 0, 128);
-        SSTableReader sstableNew = sstable(cfs, 1, 128);
-
-        // simulate tracking sstables with a failed transaction (new log file NOT deleted)
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstableNew);
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstableOld);
-
-        File tmpNewLog = copyToTmpFile(transactionLogs.getData().newLog().file);
-        File tmpOldLog = copyToTmpFile(transactionLogs.getData().oldLog().file);
-
-        Set<File> tmpFiles = new HashSet<>(TransactionLogs.getLogFiles(cfs.metadata));
-        for (String p : sstableNew.getAllFilePaths())
-            tmpFiles.add(new File(p));
-
-        sstableNew.selfRef().release();
-        sstableOld.selfRef().release();
-
-        Assert.assertEquals(tmpFiles, TransactionLogs.getTemporaryFiles(cfs.metadata, sstableNew.descriptor.directory));
-
-        // normally called at startup
-        TransactionLogs.removeUnfinishedLeftovers(cfs.metadata);
-
-        // sstable should not have been removed because the new log was found
-        Directories directories = new Directories(cfs.metadata);
-        Map<Descriptor, Set<Component>> sstables = directories.sstableLister().list();
-        assertEquals(1, sstables.size());
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstableOld.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-
-        tidier.run();
-
-        // copy old transaction files contents back or transactionlogs will throw assertions
-        Files.move(tmpNewLog.toPath(), transactionLogs.getData().newLog().file.toPath());
-        Files.move(tmpOldLog.toPath(), transactionLogs.getData().oldLog().file.toPath());
-
-        transactionLogs.close();
-    }
-
-    @Test
-    public void testRemoveUnfinishedLeftovers_oldLogFound() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstableOld = sstable(cfs, 0, 128);
-        SSTableReader sstableNew = sstable(cfs, 1, 128);
-
-        // simulate tracking sstables with a committed transaction (new log file deleted)
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        transactionLogs.trackNew(sstableNew);
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstableOld);
-
-        File tmpNewLog = copyToTmpFile(transactionLogs.getData().newLog().file);
-        File tmpOldLog = copyToTmpFile(transactionLogs.getData().oldLog().file);
-
-        transactionLogs.getData().newLog().delete(false);
-
-        Set<File> tmpFiles = new HashSet<>(TransactionLogs.getLogFiles(cfs.metadata));
-        for (String p : sstableOld.getAllFilePaths())
-            tmpFiles.add(new File(p));
-
-        sstableNew.selfRef().release();
-        sstableOld.selfRef().release();
-
-        Assert.assertEquals(tmpFiles, TransactionLogs.getTemporaryFiles(cfs.metadata, sstableOld.descriptor.directory));
-
-        // normally called at startup
-        TransactionLogs.removeUnfinishedLeftovers(cfs.metadata);
-
-        // sstable should have been removed because there was no new log.
-        Directories directories = new Directories(cfs.metadata);
-        Map<Descriptor, Set<Component>> sstables = directories.sstableLister().list();
-        assertEquals(1, sstables.size());
-
-        assertFiles(transactionLogs.getDataFolder(), new HashSet<>(sstableNew.getAllFilePaths()));
-        assertFiles(transactionLogs.getLogsFolder(), Collections.<String>emptySet());
-
-        tidier.run();
-
-        // copy old transaction files contents back or transactionlogs will throw assertions
-        Files.move(tmpNewLog.toPath(), transactionLogs.getData().newLog().file.toPath());
-        Files.move(tmpOldLog.toPath(), transactionLogs.getData().oldLog().file.toPath());
-
-        transactionLogs.close();
-    }
-
-    @Test
-    public void testGetTemporaryFiles() throws IOException
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable1 = sstable(cfs, 0, 128);
-
-        File dataFolder = sstable1.descriptor.directory;
-
-        Set<File> tmpFiles = TransactionLogs.getTemporaryFiles(cfs.metadata, dataFolder);
-        assertNotNull(tmpFiles);
-        assertEquals(0, tmpFiles.size());
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.WRITE, cfs.metadata);
-        Directories directories = new Directories(cfs.metadata);
-
-        File[] beforeSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
-
-        SSTableReader sstable2 = sstable(cfs, 1, 128);
-        transactionLogs.trackNew(sstable2);
-
-        Map<Descriptor, Set<Component>> sstables = directories.sstableLister().list();
-        assertEquals(2, sstables.size());
-
-        File[] afterSecondSSTable = dataFolder.listFiles(pathname -> !pathname.isDirectory());
-        int numNewFiles = afterSecondSSTable.length - beforeSecondSSTable.length;
-        assertTrue(numNewFiles == sstable2.getAllFilePaths().size());
-
-        tmpFiles = TransactionLogs.getTemporaryFiles(cfs.metadata, dataFolder);
-        assertNotNull(tmpFiles);
-        assertEquals(numNewFiles + 2, tmpFiles.size()); //the extra files are the transaction log files
-
-        File ssTable2DataFile = new File(sstable2.descriptor.filenameFor(Component.DATA));
-        File ssTable2IndexFile = new File(sstable2.descriptor.filenameFor(Component.PRIMARY_INDEX));
-
-        assertTrue(tmpFiles.contains(ssTable2DataFile));
-        assertTrue(tmpFiles.contains(ssTable2IndexFile));
-
-        List<File> files = directories.sstableLister().listFiles();
-        List<File> filesNoTmp = directories.sstableLister().skipTemporary(true).listFiles();
-        assertNotNull(files);
-        assertNotNull(filesNoTmp);
-
-        assertTrue(files.contains(ssTable2DataFile));
-        assertTrue(files.contains(ssTable2IndexFile));
-
-        assertFalse(filesNoTmp.contains(ssTable2DataFile));
-        assertFalse(filesNoTmp.contains(ssTable2IndexFile));
-
-        transactionLogs.finish();
-
-        //Now it should be empty since the transaction has finished
-        tmpFiles = TransactionLogs.getTemporaryFiles(cfs.metadata, dataFolder);
-        assertNotNull(tmpFiles);
-        assertEquals(0, tmpFiles.size());
-
-        filesNoTmp = directories.sstableLister().skipTemporary(true).listFiles();
-        assertNotNull(filesNoTmp);
-        assertTrue(filesNoTmp.contains(ssTable2DataFile));
-        assertTrue(filesNoTmp.contains(ssTable2IndexFile));
-
-        sstable1.selfRef().release();
-        sstable2.selfRef().release();
-    }
-
-    @Test
-    public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable
-    {
-        ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
-        SSTableReader sstable = sstable(cfs, 0, 128);
-        File dataFolder = sstable.descriptor.directory;
-
-        TransactionLogs transactionLogs = new TransactionLogs(OperationType.COMPACTION, cfs.metadata);
-        assertNotNull(transactionLogs);
-
-        TransactionLogs.SSTableTidier tidier = transactionLogs.obsoleted(sstable);
-
-        transactionLogs.finish();
-        sstable.markObsolete(tidier);
-        sstable.selfRef().release();
-
-        for (int i = 0; i < 1000; i++)
-        {
-            // This should race with the asynchronous deletion of txn log files
-            // It doesn't matter what it returns but it should not throw
-            TransactionLogs.getTemporaryFiles(cfs.metadata, dataFolder);
-        }
-    }
-
-    private static SSTableReader sstable(ColumnFamilyStore cfs, int generation, int size) throws IOException
-    {
-        Directories dir = new Directories(cfs.metadata);
-        Descriptor descriptor = new Descriptor(dir.getDirectoryForNewSSTables(), cfs.keyspace.getName(), cfs.getColumnFamilyName(), generation);
-        Set<Component> components = ImmutableSet.of(Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.TOC);
-        for (Component component : components)
-        {
-            File file = new File(descriptor.filenameFor(component));
-            file.createNewFile();
-            try (RandomAccessFile raf = new RandomAccessFile(file, "rw"))
-            {
-                raf.setLength(size);
-            }
-        }
-
-        SegmentedFile dFile = new BufferedSegmentedFile(new ChannelProxy(new File(descriptor.filenameFor(Component.DATA))), RandomAccessReader.DEFAULT_BUFFER_SIZE, 0);
-        SegmentedFile iFile = new BufferedSegmentedFile(new ChannelProxy(new File(descriptor.filenameFor(Component.PRIMARY_INDEX))), RandomAccessReader.DEFAULT_BUFFER_SIZE, 0);
-
-        SerializationHeader header = SerializationHeader.make(cfs.metadata, Collections.EMPTY_LIST);
-        StatsMetadata metadata = (StatsMetadata) new MetadataCollector(cfs.metadata.comparator)
-                                                 .finalizeMetadata(cfs.metadata.partitioner.getClass().getCanonicalName(), 0.01f, -1, header)
-                                                 .get(MetadataType.STATS);
-        SSTableReader reader = SSTableReader.internalOpen(descriptor,
-                                                          components,
-                                                          cfs.metadata,
-                                                          dFile,
-                                                          iFile,
-                                                          MockSchema.indexSummary.sharedCopy(),
-                                                          new AlwaysPresentFilter(),
-                                                          1L,
-                                                          metadata,
-                                                          SSTableReader.OpenReason.NORMAL,
-                                                          header);
-        reader.first = reader.last = MockSchema.readerBounds(generation);
-        return reader;
-    }
-
-    private static void assertFiles(String dirPath, Set<String> expectedFiles)
-    {
-        File dir = new File(dirPath);
-        for (File file : dir.listFiles())
-        {
-            if (file.isDirectory())
-                continue;
-
-            String filePath = file.getPath();
-            assertTrue(filePath, expectedFiles.contains(filePath));
-            expectedFiles.remove(filePath);
-        }
-
-        assertTrue(expectedFiles.isEmpty());
-    }
-}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
index 9a558f1..ceeb369 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterClientTest.java
@@ -99,12 +99,5 @@ public class CQLSSTableWriterClientTest
 
         File[] dataFiles = this.testDirectory.listFiles(filter);
         assertEquals(2, dataFiles.length);
-
-        File transactionsFolder = Directories.getTransactionsDirectory(testDirectory);
-        assertTrue(transactionsFolder.exists());
-
-        File[] opFiles = transactionsFolder.listFiles();
-        assertEquals(0, opFiles.length);
-
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
index 2e9768e..e7cf51c 100644
--- a/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/CQLSSTableWriterTest.java
@@ -45,7 +45,6 @@ import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.OutputHandler;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 
 public class CQLSSTableWriterTest
 {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
index fd801ad..d9516cb 100644
--- a/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/SSTableRewriterTest.java
@@ -44,7 +44,7 @@ import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.RowUpdateBuilder;
 import org.apache.cassandra.db.SerializationHeader;
 import org.apache.cassandra.db.rows.EncodingStats;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.db.rows.UnfilteredRowIterator;
 import org.apache.cassandra.db.compaction.AbstractCompactionStrategy;
 import org.apache.cassandra.db.compaction.CompactionController;
@@ -109,7 +109,7 @@ public class SSTableRewriterTest extends SchemaLoader
         Keyspace keyspace = Keyspace.open(KEYSPACE);
         ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF);
         store.truncateBlocking();
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
     }
 
     @Test
@@ -145,7 +145,7 @@ public class SSTableRewriterTest extends SchemaLoader
             }
             writer.finish();
         }
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         validateCFS(cfs);
         int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
         assertEquals(1, filecounts);
@@ -177,7 +177,7 @@ public class SSTableRewriterTest extends SchemaLoader
             }
             writer.finish();
         }
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         validateCFS(cfs);
         int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
         assertEquals(1, filecounts);
@@ -232,7 +232,7 @@ public class SSTableRewriterTest extends SchemaLoader
             assertTrue(checked);
             writer.finish();
         }
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         validateCFS(cfs);
         int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
         assertEquals(1, filecounts);
@@ -277,12 +277,12 @@ public class SSTableRewriterTest extends SchemaLoader
             // open till .abort() is called (via the builder)
             if (!FBUtilities.isWindows())
             {
-                TransactionLogs.waitForDeletions();
+                TransactionLog.waitForDeletions();
                 assertFileCounts(dir.list());
             }
             writer.abort();
             txn.abort();
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             int datafiles = assertFileCounts(dir.list());
             assertEquals(datafiles, 0);
             validateCFS(cfs);
@@ -328,7 +328,7 @@ public class SSTableRewriterTest extends SchemaLoader
             sstables = rewriter.finish();
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         long sum = 0;
         for (SSTableReader x : cfs.getLiveSSTables())
@@ -337,7 +337,7 @@ public class SSTableRewriterTest extends SchemaLoader
         assertEquals(startStorageMetricsLoad - sBytesOnDisk + sum, StorageMetrics.load.getCount());
         assertEquals(files, sstables.size());
         assertEquals(files, cfs.getLiveSSTables().size());
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         // tmplink and tmp files should be gone:
         assertEquals(sum, cfs.metric.totalDiskSpaceUsed.getCount());
@@ -382,7 +382,7 @@ public class SSTableRewriterTest extends SchemaLoader
 
         assertEquals(files, sstables.size());
         assertEquals(files, cfs.getLiveSSTables().size());
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         assertFileCounts(s.descriptor.directory.list());
         validateCFS(cfs);
@@ -519,7 +519,7 @@ public class SSTableRewriterTest extends SchemaLoader
             test.run(scanner, controller, s, cfs, rewriter, txn);
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         assertEquals(startSize, cfs.metric.liveDiskSpaceUsed.getCount());
         assertEquals(1, cfs.getLiveSSTables().size());
@@ -567,7 +567,7 @@ public class SSTableRewriterTest extends SchemaLoader
             }
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         assertEquals(files - 1, cfs.getLiveSSTables().size()); // we never wrote anything to the last file
         assertFileCounts(s.descriptor.directory.list());
@@ -609,7 +609,7 @@ public class SSTableRewriterTest extends SchemaLoader
             sstables = rewriter.finish();
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         assertFileCounts(s.descriptor.directory.list());
         validateCFS(cfs);
     }
@@ -650,7 +650,7 @@ public class SSTableRewriterTest extends SchemaLoader
         }
         assertEquals(files, sstables.size());
         assertEquals(files, cfs.getLiveSSTables().size());
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         assertFileCounts(s.descriptor.directory.list());
 
         validateCFS(cfs);
@@ -670,7 +670,7 @@ public class SSTableRewriterTest extends SchemaLoader
             splitter.split();
 
             assertFileCounts(s.descriptor.directory.list());
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
 
             for (File f : s.descriptor.directory.listFiles())
             {
@@ -746,7 +746,7 @@ public class SSTableRewriterTest extends SchemaLoader
                 s.selfRef().release();
         }
 
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         int filecount = assertFileCounts(s.descriptor.directory.list());
         assertEquals(filecount, 1);
@@ -825,7 +825,7 @@ public class SSTableRewriterTest extends SchemaLoader
             rewriter.finish();
         }
         validateKeys(keyspace);
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         validateCFS(cfs);
         truncate(cfs);
     }
@@ -923,7 +923,7 @@ public class SSTableRewriterTest extends SchemaLoader
     public static void truncate(ColumnFamilyStore cfs)
     {
         cfs.truncateBlocking();
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         Uninterruptibles.sleepUninterruptibly(10L, TimeUnit.MILLISECONDS);
         assertEquals(0, cfs.metric.liveDiskSpaceUsed.getCount());
         assertEquals(0, cfs.metric.totalDiskSpaceUsed.getCount());

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
index c763932..0898515 100644
--- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
+++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java
@@ -51,17 +51,10 @@ public class MetadataSerializerTest
 
         CFMetaData cfm = SchemaLoader.standardCFMD("ks1", "cf1");
 
-
         ReplayPosition rp = new ReplayPosition(11L, 12);
 
-
         MetadataCollector collector = new MetadataCollector(cfm.comparator).replayPosition(rp);
 
-
-        Set<Integer> ancestors = Sets.newHashSet(1, 2, 3, 4);
-        for (int i : ancestors)
-            collector.addAncestor(i);
-
         String partitioner = RandomPartitioner.class.getCanonicalName();
         double bfFpChance = 0.1;
         Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, SerializationHeader.make(cfm, Collections.EMPTY_LIST));

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/schema/DefsTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/schema/DefsTest.java b/test/unit/org/apache/cassandra/schema/DefsTest.java
index 680c016..98a954c 100644
--- a/test/unit/org/apache/cassandra/schema/DefsTest.java
+++ b/test/unit/org/apache/cassandra/schema/DefsTest.java
@@ -39,8 +39,9 @@ import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.cql3.QueryProcessor;
 import org.apache.cassandra.cql3.UntypedResultSet;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.db.marshal.BytesType;
 import org.apache.cassandra.db.marshal.UTF8Type;
 import org.apache.cassandra.exceptions.ConfigurationException;
@@ -205,7 +206,7 @@ public class DefsTest
         ColumnFamilyStore store = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
         assertNotNull(store);
         store.forceBlockingFlush();
-        assertTrue(store.directories.sstableLister().list().size() > 0);
+        assertTrue(store.directories.sstableLister(Directories.OnTxnErr.THROW).list().size() > 0);
 
         MigrationManager.announceColumnFamilyDrop(ks.name, cfm.cfName);
 
@@ -227,7 +228,7 @@ public class DefsTest
 
         // verify that the files are gone.
         Supplier<Object> lambda = () -> {
-            for (File file : store.directories.sstableLister().listFiles())
+            for (File file : store.directories.sstableLister(Directories.OnTxnErr.THROW).listFiles())
             {
                 if (file.getPath().endsWith("Data.db") && !new File(file.getPath().replace("Data.db", "Compacted")).exists())
                     return false;
@@ -276,7 +277,7 @@ public class DefsTest
         ColumnFamilyStore cfs = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName);
         assertNotNull(cfs);
         cfs.forceBlockingFlush();
-        assertTrue(!cfs.directories.sstableLister().list().isEmpty());
+        assertTrue(!cfs.directories.sstableLister(Directories.OnTxnErr.THROW).list().isEmpty());
 
         MigrationManager.announceKeyspaceDrop(ks.name);
 
@@ -521,7 +522,7 @@ public class DefsTest
 
         // check
         assertTrue(cfs.indexManager.getIndexes().isEmpty());
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
         assertFalse(new File(desc.filenameFor(Component.DATA)).exists());
     }
 


[3/7] cassandra git commit: Improve transaction log under FS corruption

Posted by be...@apache.org.
Improve transaction log under FS corruption

To address concerns with recovery from file system
failures, we consolidate the new and old log
files into a single file, introduce checksums to the
file, and record on-disk sstable state to permit
validation of the log file on restart. This permits
us to be pessimistic in the case of any failure,
leaving all affected files on disk.

patch by stefania; reviewed by benedict for CASSANDRA-7066


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5726625a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5726625a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5726625a

Branch: refs/heads/cassandra-3.0
Commit: 5726625a5c7cf47a67509540f7146d05b668bc20
Parents: 47b66d7
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Wed Jul 29 12:01:01 2015 +0800
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Mon Aug 17 10:20:01 2015 +0100

----------------------------------------------------------------------
 NEWS.txt                                        |   18 +
 bin/sstablelister                               |   55 -
 bin/sstablelister.bat                           |   41 -
 bin/sstableutil                                 |   55 +
 bin/sstableutil.bat                             |   41 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |   12 +-
 .../org/apache/cassandra/db/Directories.java    |  145 ++-
 .../db/compaction/CompactionManagerMBean.java   |    3 +-
 .../cassandra/db/compaction/Upgrader.java       |    9 -
 .../writers/MajorLeveledCompactionWriter.java   |    9 +-
 .../apache/cassandra/db/lifecycle/Helpers.java  |   12 +-
 .../db/lifecycle/LifecycleTransaction.java      |   69 +-
 .../apache/cassandra/db/lifecycle/Tracker.java  |    4 +-
 .../cassandra/db/lifecycle/TransactionLog.java  | 1137 ++++++++++++++++++
 .../cassandra/db/lifecycle/TransactionLogs.java |  786 ------------
 .../cassandra/io/sstable/SSTableLoader.java     |  162 ++-
 .../io/sstable/format/SSTableReader.java        |   22 +-
 .../cassandra/io/sstable/format/Version.java    |    2 +
 .../io/sstable/format/big/BigFormat.java        |   12 +
 .../io/sstable/metadata/CompactionMetadata.java |   43 +-
 .../metadata/LegacyMetadataSerializer.java      |   13 +-
 .../io/sstable/metadata/MetadataCollector.java  |   28 +-
 .../org/apache/cassandra/io/util/FileUtils.java |    5 -
 .../apache/cassandra/service/GCInspector.java   |    4 +-
 .../apache/cassandra/service/StartupChecks.java |    3 +-
 .../cassandra/service/StorageService.java       |    4 +-
 .../cassandra/tools/SSTableExpiredBlockers.java |    3 +-
 .../cassandra/tools/SSTableLevelResetter.java   |    3 +-
 .../cassandra/tools/SSTableMetadataViewer.java  |    2 -
 .../cassandra/tools/SSTableOfflineRelevel.java  |    2 +-
 .../cassandra/tools/StandaloneLister.java       |  214 ----
 .../cassandra/tools/StandaloneSSTableUtil.java  |  241 ++++
 .../cassandra/tools/StandaloneScrubber.java     |    6 +-
 .../cassandra/tools/StandaloneSplitter.java     |    4 +-
 .../cassandra/tools/StandaloneUpgrader.java     |    6 +-
 .../cassandra/tools/StandaloneVerifier.java     |    2 +-
 .../apache/cassandra/tools/nodetool/Stop.java   |    2 +-
 .../cassandra/db/ColumnFamilyStoreTest.java     |    2 +-
 .../apache/cassandra/db/DirectoriesTest.java    |    7 +-
 .../org/apache/cassandra/db/KeyCacheTest.java   |    6 +-
 .../unit/org/apache/cassandra/db/ScrubTest.java |    4 +-
 .../cassandra/db/lifecycle/HelpersTest.java     |    9 +-
 .../db/lifecycle/LifecycleTransactionTest.java  |    2 +-
 .../db/lifecycle/RealTransactionsTest.java      |   15 +-
 .../cassandra/db/lifecycle/TrackerTest.java     |   90 +-
 .../db/lifecycle/TransactionLogTest.java        |  791 ++++++++++++
 .../db/lifecycle/TransactionLogsTest.java       |  581 ---------
 .../io/sstable/CQLSSTableWriterClientTest.java  |    7 -
 .../io/sstable/CQLSSTableWriterTest.java        |    1 -
 .../io/sstable/SSTableRewriterTest.java         |   36 +-
 .../metadata/MetadataSerializerTest.java        |    7 -
 .../org/apache/cassandra/schema/DefsTest.java   |   11 +-
 52 files changed, 2625 insertions(+), 2123 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index ef764ad..365ed31 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -38,6 +38,24 @@ Upgrading
    - The default JVM flag -XX:+PerfDisableSharedMem will cause the following tools JVM
      to stop working: jps, jstack, jinfo, jmc, jcmd as well as 3rd party tools like Jolokia.
      If you wish to use these tools you can comment this flag out in cassandra-env.{sh,ps1}
+   - New transaction log files have been introduced to replace the compactions_in_progress
+     system table, temporary file markers (tmp and tmplink) and sstable ancerstors.
+     Therefore, compaction metadata no longer contains ancestors. Transaction log files
+     list sstable descriptors involved in compactions and other operations such as flushing
+     and streaming. Use the sstableutil tool to list any sstable files currently involved
+     in operations not yet completed, which previously would have been marked as temporary.
+     A transaction log file contains one sstable per line, with the prefix "add:" or "remove:".
+     They also contain a special line "commit", only inserted at the end when the transaction
+     is committed. On startup we use these files to cleanup any partial transactions that were
+     in progress when the process exited. If the commit line is found, we keep new sstables
+     (those with the "add" prefix) and delete the old sstables (those with the "remove" prefix),
+     vice-versa if the commit line is missing. Should you lose or delete these log files,
+     both old and new sstable files will be kept as live files, which will result in duplicated
+     sstables. These files are protected by incremental checksums so you should not manually
+     edit them. When restoring a full backup or moving sstable files, you should clean-up
+     any left over transactions and their temporary files first. You can use this command:
+      ===> sstableutil -c ks table
+     See CASSANDRA-7066 for full details.
    - New write stages have been added for batchlog and materialized view mutations
      you can set their size in cassandra.yaml
    - User defined functions are now executed in a sandbox.

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/bin/sstablelister
----------------------------------------------------------------------
diff --git a/bin/sstablelister b/bin/sstablelister
deleted file mode 100755
index a79409d..0000000
--- a/bin/sstablelister
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
-    for include in /usr/share/cassandra/cassandra.in.sh \
-                   /usr/local/share/cassandra/cassandra.in.sh \
-                   /opt/cassandra/cassandra.in.sh \
-                   ~/.cassandra.in.sh \
-                   "`dirname "$0"`/cassandra.in.sh"; do
-        if [ -r "$include" ]; then
-            . "$include"
-            break
-        fi
-    done
-elif [ -r "$CASSANDRA_INCLUDE" ]; then
-    . "$CASSANDRA_INCLUDE"
-fi
-
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
-    JAVA="$JAVA_HOME/bin/java"
-else
-    JAVA="`which java`"
-fi
-
-if [ -z "$CLASSPATH" ]; then
-    echo "You must set the CLASSPATH var" >&2
-    exit 1
-fi
-
-if [ "x$MAX_HEAP_SIZE" = "x" ]; then
-    MAX_HEAP_SIZE="256M"
-fi
-
-"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
-        -Dcassandra.storagedir="$cassandra_storagedir" \
-        -Dlogback.configurationFile=logback-tools.xml \
-        org.apache.cassandra.tools.StandaloneLister "$@"
-
-# vi:ai sw=4 ts=4 tw=0 et

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/bin/sstablelister.bat
----------------------------------------------------------------------
diff --git a/bin/sstablelister.bat b/bin/sstablelister.bat
deleted file mode 100644
index cb50a08..0000000
--- a/bin/sstablelister.bat
+++ /dev/null
@@ -1,41 +0,0 @@
-@REM
-@REM  Licensed to the Apache Software Foundation (ASF) under one or more
-@REM  contributor license agreements.  See the NOTICE file distributed with
-@REM  this work for additional information regarding copyright ownership.
-@REM  The ASF licenses this file to You under the Apache License, Version 2.0
-@REM  (the "License"); you may not use this file except in compliance with
-@REM  the License.  You may obtain a copy of the License at
-@REM
-@REM      http://www.apache.org/licenses/LICENSE-2.0
-@REM
-@REM  Unless required by applicable law or agreed to in writing, software
-@REM  distributed under the License is distributed on an "AS IS" BASIS,
-@REM  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@REM  See the License for the specific language governing permissions and
-@REM  limitations under the License.
-
-@echo off
-if "%OS%" == "Windows_NT" setlocal
-
-pushd "%~dp0"
-call cassandra.in.bat
-
-if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.StandaloneLister
-if NOT DEFINED JAVA_HOME goto :err
-
-REM ***** JAVA options *****
-set JAVA_OPTS=^
- -Dlogback.configurationFile=logback-tools.xml
-
-set TOOLS_PARAMS=
-
-"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %*
-goto finally
-
-:err
-echo JAVA_HOME environment variable must be set!
-pause
-
-:finally
-
-ENDLOCAL

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/bin/sstableutil
----------------------------------------------------------------------
diff --git a/bin/sstableutil b/bin/sstableutil
new file mode 100755
index 0000000..2860729
--- /dev/null
+++ b/bin/sstableutil
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+    for include in /usr/share/cassandra/cassandra.in.sh \
+                   /usr/local/share/cassandra/cassandra.in.sh \
+                   /opt/cassandra/cassandra.in.sh \
+                   ~/.cassandra.in.sh \
+                   "`dirname "$0"`/cassandra.in.sh"; do
+        if [ -r "$include" ]; then
+            . "$include"
+            break
+        fi
+    done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+    . "$CASSANDRA_INCLUDE"
+fi
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -x "$JAVA_HOME/bin/java" ]; then
+    JAVA="$JAVA_HOME/bin/java"
+else
+    JAVA="`which java`"
+fi
+
+if [ -z "$CLASSPATH" ]; then
+    echo "You must set the CLASSPATH var" >&2
+    exit 1
+fi
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+    MAX_HEAP_SIZE="256M"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+        -Dcassandra.storagedir="$cassandra_storagedir" \
+        -Dlogback.configurationFile=logback-tools.xml \
+        org.apache.cassandra.tools.StandaloneSSTableUtil "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/bin/sstableutil.bat
----------------------------------------------------------------------
diff --git a/bin/sstableutil.bat b/bin/sstableutil.bat
new file mode 100644
index 0000000..bc3eb8a
--- /dev/null
+++ b/bin/sstableutil.bat
@@ -0,0 +1,41 @@
+@REM
+@REM  Licensed to the Apache Software Foundation (ASF) under one or more
+@REM  contributor license agreements.  See the NOTICE file distributed with
+@REM  this work for additional information regarding copyright ownership.
+@REM  The ASF licenses this file to You under the Apache License, Version 2.0
+@REM  (the "License"); you may not use this file except in compliance with
+@REM  the License.  You may obtain a copy of the License at
+@REM
+@REM      http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM  Unless required by applicable law or agreed to in writing, software
+@REM  distributed under the License is distributed on an "AS IS" BASIS,
+@REM  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM  See the License for the specific language governing permissions and
+@REM  limitations under the License.
+
+@echo off
+if "%OS%" == "Windows_NT" setlocal
+
+pushd "%~dp0"
+call cassandra.in.bat
+
+if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.StandaloneSSTableUtil
+if NOT DEFINED JAVA_HOME goto :err
+
+REM ***** JAVA options *****
+set JAVA_OPTS=^
+ -Dlogback.configurationFile=logback-tools.xml
+
+set TOOLS_PARAMS=
+
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %*
+goto finally
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+:finally
+
+ENDLOCAL

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index c7d12a2..202047b 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -353,7 +353,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
         if (data.loadsstables)
         {
-            Directories.SSTableLister sstableFiles = directories.sstableLister().skipTemporary(true);
+            Directories.SSTableLister sstableFiles = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
             Collection<SSTableReader> sstables = SSTableReader.openAll(sstableFiles.list().entrySet(), metadata);
             data.addInitialSSTables(sstables);
         }
@@ -459,7 +459,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         SystemKeyspace.removeTruncationRecord(metadata.cfId);
 
         data.dropSSTables();
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         indexManager.invalidate();
         materializedViewManager.invalidate();
@@ -503,7 +503,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     {
         // get the max generation number, to prevent generation conflicts
         Directories directories = new Directories(metadata);
-        Directories.SSTableLister lister = directories.sstableLister().includeBackups(true);
+        Directories.SSTableLister lister = directories.sstableLister(Directories.OnTxnErr.IGNORE).includeBackups(true);
         List<Integer> generations = new ArrayList<Integer>();
         for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
         {
@@ -534,7 +534,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         LifecycleTransaction.removeUnfinishedLeftovers(metadata);
 
         logger.debug("Further extra check for orphan sstable files for {}", metadata.cfName);
-        for (Map.Entry<Descriptor,Set<Component>> sstableFiles : directories.sstableLister().list().entrySet())
+        for (Map.Entry<Descriptor,Set<Component>> sstableFiles : directories.sstableLister(Directories.OnTxnErr.IGNORE).list().entrySet())
         {
             Descriptor desc = sstableFiles.getKey();
             Set<Component> components = sstableFiles.getValue();
@@ -642,7 +642,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
             currentDescriptors.add(sstable.descriptor);
         Set<SSTableReader> newSSTables = new HashSet<>();
 
-        Directories.SSTableLister lister = directories.sstableLister().skipTemporary(true);
+        Directories.SSTableLister lister = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
         for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
         {
             Descriptor descriptor = entry.getKey();
@@ -1644,7 +1644,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         Map<Integer, SSTableReader> active = new HashMap<>();
         for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
             active.put(sstable.descriptor.generation, sstable);
-        Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
+        Map<Descriptor, Set<Component>> snapshots = directories.sstableLister(Directories.OnTxnErr.IGNORE).snapshots(tag).list();
         Refs<SSTableReader> refs = new Refs<>();
         try
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index bede4c4..fa01269 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -31,6 +31,7 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.util.*;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
@@ -92,7 +93,6 @@ public class Directories
 
     public static final String BACKUPS_SUBDIR = "backups";
     public static final String SNAPSHOT_SUBDIR = "snapshots";
-    public static final String TRANSACTIONS_SUBDIR = "transactions";
     public static final String SECONDARY_INDEX_NAME_SEPARATOR = ".";
 
     public static final DataDirectory[] dataDirectories;
@@ -142,7 +142,7 @@ public class Directories
     {
         X, W, XW, R, XR, RW, XRW;
 
-        private FileAction()
+        FileAction()
         {
         }
 
@@ -468,40 +468,6 @@ public class Directories
         }
     }
 
-    public static File getTransactionsDirectory(File folder)
-    {
-        return getOrCreate(folder, TRANSACTIONS_SUBDIR);
-    }
-
-    public List<File> getExistingDirectories(String subFolder)
-    {
-        List<File> ret = new ArrayList<>();
-        for (File dir : dataPaths)
-        {
-            File subDir = getExistingDirectory(dir, subFolder);
-            if (subDir != null)
-                ret.add(subDir);
-
-        }
-        return ret;
-    }
-
-    public static File getExistingDirectory(File folder, String subFolder)
-    {
-        File subDir = new File(folder, join(subFolder));
-        if (subDir.exists())
-        {
-            assert(subDir.isDirectory());
-            return subDir;
-        }
-        return null;
-    }
-
-    public SSTableLister sstableLister()
-    {
-        return new SSTableLister();
-    }
-
     public static class DataDirectory
     {
         public final File location;
@@ -549,10 +515,42 @@ public class Directories
         }
     }
 
+    /** The type of files that can be listed by SSTableLister, we never return txn logs,
+     * use LifecycleTransaction.getFiles() if you need txn logs. */
+    public enum FileType
+    {
+        /** A permanent sstable file that is safe to use. */
+        FINAL,
+
+        /** A temporary sstable file that will soon be deleted. */
+        TEMPORARY,
+
+        /** A transaction log file (contains information on final and temporary files). */
+        TXN_LOG
+    }
+
+    /**
+     * How to handle a failure to read a txn log file. Note that we will try a few
+     * times before giving up.
+     **/
+    public enum OnTxnErr
+    {
+        /** Throw the exception */
+        THROW,
+
+        /** Ignore the txn log file */
+        IGNORE
+    }
+
+    public SSTableLister sstableLister(OnTxnErr onTxnErr)
+    {
+        return new SSTableLister(onTxnErr);
+    }
+
     public class SSTableLister
     {
+        private final OnTxnErr onTxnErr;
         private boolean skipTemporary;
-        private boolean onlyTemporary;
         private boolean includeBackups;
         private boolean onlyBackups;
         private int nbFiles;
@@ -560,19 +558,16 @@ public class Directories
         private boolean filtered;
         private String snapshotName;
 
-        public SSTableLister skipTemporary(boolean b)
+        private SSTableLister(OnTxnErr onTxnErr)
         {
-            if (filtered)
-                throw new IllegalStateException("list() has already been called");
-            skipTemporary = b;
-            return this;
+            this.onTxnErr = onTxnErr;
         }
 
-        public SSTableLister onlyTemporary(boolean b)
+        public SSTableLister skipTemporary(boolean b)
         {
             if (filtered)
                 throw new IllegalStateException("list() has already been called");
-            onlyTemporary = b;
+            skipTemporary = b;
             return this;
         }
 
@@ -633,56 +628,54 @@ public class Directories
 
                 if (snapshotName != null)
                 {
-                    getSnapshotDirectory(location, snapshotName).listFiles(getFilter(location));
+                    LifecycleTransaction.getFiles(getSnapshotDirectory(location, snapshotName).toPath(), getFilter(), onTxnErr);
                     continue;
                 }
 
                 if (!onlyBackups)
-                    location.listFiles(getFilter(location));
+                    LifecycleTransaction.getFiles(location.toPath(), getFilter(), onTxnErr);
 
                 if (includeBackups)
-                    getBackupsDirectory(location).listFiles(getFilter(location));
+                    LifecycleTransaction.getFiles(getBackupsDirectory(location).toPath(), getFilter(), onTxnErr);
             }
+
             filtered = true;
         }
 
-        private FileFilter getFilter(File location)
+        private BiFunction<File, FileType, Boolean> getFilter()
         {
-           final Set<File> temporaryFiles = skipTemporary || onlyTemporary
-                                            ? LifecycleTransaction.getTemporaryFiles(metadata, location)
-                                            : Collections.<File>emptySet();
-
-            return new FileFilter()
+            // This function always return false since it adds to the components map
+            return (file, type) ->
             {
-                // This function always return false since accepts adds to the components map
-                public boolean accept(File file)
+                switch (type)
                 {
-                    if (file.isDirectory())
-                        return false;
-
-                    Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
-                    if (pair == null)
+                    case TXN_LOG:
                         return false;
+                    case TEMPORARY:
+                        if (skipTemporary)
+                            return false;
 
-                    // we are only interested in the SSTable files that belong to the specific ColumnFamily
-                    if (!pair.left.ksname.equals(metadata.ksName) || !pair.left.cfname.equals(metadata.cfName))
-                        return false;
+                    case FINAL:
+                        Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
+                        if (pair == null)
+                            return false;
 
-                    if (skipTemporary && temporaryFiles.contains(file))
-                        return false;
+                        // we are only interested in the SSTable files that belong to the specific ColumnFamily
+                        if (!pair.left.ksname.equals(metadata.ksName) || !pair.left.cfname.equals(metadata.cfName))
+                            return false;
 
-                    if (onlyTemporary && !temporaryFiles.contains(file))
+                        Set<Component> previous = components.get(pair.left);
+                        if (previous == null)
+                        {
+                            previous = new HashSet<>();
+                            components.put(pair.left, previous);
+                        }
+                        previous.add(pair.right);
+                        nbFiles++;
                         return false;
 
-                    Set<Component> previous = components.get(pair.left);
-                    if (previous == null)
-                    {
-                        previous = new HashSet<>();
-                        components.put(pair.left, previous);
-                    }
-                    previous.add(pair.right);
-                    nbFiles++;
-                    return false;
+                    default:
+                        throw new AssertionError();
                 }
             };
         }
@@ -917,7 +910,7 @@ public class Directories
         {
             super();
             Builder<String> builder = ImmutableSet.builder();
-            for (File file : sstableLister().listFiles())
+            for (File file : sstableLister(Directories.OnTxnErr.THROW).listFiles())
                 builder.add(file.getName());
             alive = builder.build();
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
index 8e200a1..d5da0fe 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
@@ -58,7 +58,8 @@ public interface CompactionManagerMBean
     /**
      * Stop an individual running compaction using the compactionId.
      * @param compactionId Compaction ID of compaction to stop. Such IDs can be found in
-     *                     the compactions_in_progress table of the system keyspace.
+     *                     the transaction log files whose name starts with compaction_,
+     *                     located in the table transactions folder.
      */
     public void stopCompactionById(String compactionId);
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
index 5a36210..ebfd997 100644
--- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
+++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
@@ -69,15 +69,6 @@ public class Upgrader
     private SSTableWriter createCompactionWriter(long repairedAt)
     {
         MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator());
-
-        // Get the max timestamp of the precompacted sstables
-        // and adds generation of live ancestors
-        sstableMetadataCollector.addAncestor(sstable.descriptor.generation);
-        for (Integer i : sstable.getAncestors())
-        {
-            if (new File(sstable.descriptor.withGeneration(i).filenameFor(Component.DATA)).exists())
-                sstableMetadataCollector.addAncestor(i);
-        }
         sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel());
         return SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(directory)),
                                     estimatedRows,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index a826809..73ce216 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -45,7 +45,6 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
     private long partitionsWritten = 0;
     private long totalWrittenInLevel = 0;
     private int sstablesWritten = 0;
-    private final boolean skipAncestors;
 
     public MajorLeveledCompactionWriter(ColumnFamilyStore cfs,
                                         LifecycleTransaction txn,
@@ -70,17 +69,13 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
         long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(nonExpiredSSTables) / maxSSTableSize);
         long keysPerSSTable = estimatedTotalKeys / estimatedSSTables;
         File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
-        skipAncestors = estimatedSSTables * allSSTables.size() > 200000; // magic number, avoid storing too much ancestor information since allSSTables are ancestors to *all* resulting sstables
-
-        if (skipAncestors)
-            logger.warn("Many sstables involved in compaction, skipping storing ancestor information to avoid running out of memory");
 
         @SuppressWarnings("resource")
         SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(sstableDirectory)),
                                                     keysPerSSTable,
                                                     minRepairedAt,
                                                     cfs.metadata,
-                                                    new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel, skipAncestors),
+                                                    new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel),
                                                     SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
                                                     txn);
         sstableWriter.switchWriter(writer);
@@ -108,7 +103,7 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
                                                         averageEstimatedKeysPerSSTable,
                                                         minRepairedAt,
                                                         cfs.metadata,
-                                                        new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel, skipAncestors),
+                                                        new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel),
                                                         SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
                                                         txn);
             sstableWriter.switchWriter(writer);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
index 8cb92b9..98983c5 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
@@ -106,12 +106,12 @@ class Helpers
             assert !reader.isReplaced();
     }
 
-    static Throwable markObsolete(List<TransactionLogs.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable markObsolete(List<TransactionLog.Obsoletion> obsoletions, Throwable accumulate)
     {
         if (obsoletions == null || obsoletions.isEmpty())
             return accumulate;
 
-        for (TransactionLogs.Obsoletion obsoletion : obsoletions)
+        for (TransactionLog.Obsoletion obsoletion : obsoletions)
         {
             try
             {
@@ -125,13 +125,13 @@ class Helpers
         return accumulate;
     }
 
-    static Throwable prepareForObsoletion(Iterable<SSTableReader> readers, TransactionLogs txnLogs, List<TransactionLogs.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable prepareForObsoletion(Iterable<SSTableReader> readers, TransactionLog txnLogs, List<TransactionLog.Obsoletion> obsoletions, Throwable accumulate)
     {
         for (SSTableReader reader : readers)
         {
             try
             {
-                obsoletions.add(new TransactionLogs.Obsoletion(reader, txnLogs.obsoleted(reader)));
+                obsoletions.add(new TransactionLog.Obsoletion(reader, txnLogs.obsoleted(reader)));
             }
             catch (Throwable t)
             {
@@ -141,12 +141,12 @@ class Helpers
         return accumulate;
     }
 
-    static Throwable abortObsoletion(List<TransactionLogs.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable abortObsoletion(List<TransactionLog.Obsoletion> obsoletions, Throwable accumulate)
     {
         if (obsoletions == null || obsoletions.isEmpty())
             return accumulate;
 
-        for (TransactionLogs.Obsoletion obsoletion : obsoletions)
+        for (TransactionLog.Obsoletion obsoletion : obsoletions)
         {
             try
             {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index 5bda4d4..c6cb979 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -18,10 +18,11 @@
 package org.apache.cassandra.db.lifecycle;
 
 import java.io.File;
+import java.nio.file.Path;
 import java.util.*;
+import java.util.function.BiFunction;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
 import com.google.common.base.Predicate;
 import com.google.common.collect.*;
 
@@ -29,6 +30,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -96,7 +98,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
 
     public final Tracker tracker;
     // The transaction logs keep track of new and old sstable files
-    private final TransactionLogs transactionLogs;
+    private final TransactionLog transactionLog;
     // the original readers this transaction was opened over, and that it guards
     // (no other transactions may operate over these readers concurrently)
     private final Set<SSTableReader> originals = new HashSet<>();
@@ -113,7 +115,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
     private final State staged = new State();
 
     // the tidier and their readers, to be used for marking readers obsoleted during a commit
-    private List<TransactionLogs.Obsoletion> obsoletions;
+    private List<TransactionLog.Obsoletion> obsoletions;
 
     /**
      * construct a Transaction for use in an offline operation
@@ -141,7 +143,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
     public static LifecycleTransaction offline(OperationType operationType, CFMetaData metadata)
     {
         Tracker dummy = new Tracker(null, false);
-        return new LifecycleTransaction(dummy, new TransactionLogs(operationType, metadata, dummy), Collections.emptyList());
+        return new LifecycleTransaction(dummy, new TransactionLog(operationType, metadata, dummy), Collections.emptyList());
     }
 
     /**
@@ -150,18 +152,18 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
     public static LifecycleTransaction offline(OperationType operationType, File operationFolder)
     {
         Tracker dummy = new Tracker(null, false);
-        return new LifecycleTransaction(dummy, new TransactionLogs(operationType, operationFolder, dummy), Collections.emptyList());
+        return new LifecycleTransaction(dummy, new TransactionLog(operationType, operationFolder, dummy), Collections.emptyList());
     }
 
     LifecycleTransaction(Tracker tracker, OperationType operationType, Iterable<SSTableReader> readers)
     {
-        this(tracker, new TransactionLogs(operationType, getMetadata(tracker, readers), tracker), readers);
+        this(tracker, new TransactionLog(operationType, getMetadata(tracker, readers), tracker), readers);
     }
 
-    LifecycleTransaction(Tracker tracker, TransactionLogs transactionLogs, Iterable<SSTableReader> readers)
+    LifecycleTransaction(Tracker tracker, TransactionLog transactionLog, Iterable<SSTableReader> readers)
     {
         this.tracker = tracker;
-        this.transactionLogs = transactionLogs;
+        this.transactionLog = transactionLog;
         for (SSTableReader reader : readers)
         {
             originals.add(reader);
@@ -185,19 +187,19 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
         return null;
     }
 
-    public TransactionLogs logs()
+    public TransactionLog log()
     {
-        return transactionLogs;
+        return transactionLog;
     }
 
     public OperationType opType()
     {
-        return transactionLogs.getType();
+        return transactionLog.getType();
     }
 
     public UUID opId()
     {
-        return transactionLogs.getId();
+        return transactionLog.getId();
     }
 
     public void doPrepare()
@@ -210,8 +212,8 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
 
         // prepare for compaction obsolete readers as long as they were part of the original set
         // since those that are not original are early readers that share the same desc with the finals
-        maybeFail(prepareForObsoletion(filterIn(logged.obsolete, originals), transactionLogs, obsoletions = new ArrayList<>(), null));
-        transactionLogs.prepareToCommit();
+        maybeFail(prepareForObsoletion(filterIn(logged.obsolete, originals), transactionLog, obsoletions = new ArrayList<>(), null));
+        transactionLog.prepareToCommit();
     }
 
     /**
@@ -226,7 +228,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
         maybeFail(accumulate);
 
         // transaction log commit failure means we must abort; safe commit is not possible
-        maybeFail(transactionLogs.commit(null));
+        maybeFail(transactionLog.commit(null));
 
         // this is now the point of no return; we cannot safely rollback, so we ignore exceptions until we're done
         // we restore state by obsoleting our obsolete files, releasing our references to them, and updating our size
@@ -235,7 +237,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
         accumulate = markObsolete(obsoletions, accumulate);
         accumulate = tracker.updateSizeTracking(logged.obsolete, logged.update, accumulate);
         accumulate = release(selfRefs(logged.obsolete), accumulate);
-        accumulate = tracker.notifySSTablesChanged(originals, logged.update, transactionLogs.getType(), accumulate);
+        accumulate = tracker.notifySSTablesChanged(originals, logged.update, transactionLog.getType(), accumulate);
 
         return accumulate;
     }
@@ -251,16 +253,16 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
         accumulate = abortObsoletion(obsoletions, accumulate);
 
         if (logged.isEmpty() && staged.isEmpty())
-            return transactionLogs.abort(accumulate);
+            return transactionLog.abort(accumulate);
 
         // mark obsolete all readers that are not versions of those present in the original set
         Iterable<SSTableReader> obsolete = filterOut(concatUniq(staged.update, logged.update), originals);
         logger.debug("Obsoleting {}", obsolete);
 
-        accumulate = prepareForObsoletion(obsolete, transactionLogs, obsoletions = new ArrayList<>(), accumulate);
+        accumulate = prepareForObsoletion(obsolete, transactionLog, obsoletions = new ArrayList<>(), accumulate);
         // it's safe to abort even if committed, see maybeFail in doCommit() above, in this case it will just report
         // a failure to abort, which is useful information to have for debug
-        accumulate = transactionLogs.abort(accumulate);
+        accumulate = transactionLog.abort(accumulate);
         accumulate = markObsolete(obsoletions, accumulate);
 
         // replace all updated readers with a version restored to its original state
@@ -491,7 +493,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
             originals.remove(reader);
             marked.remove(reader);
         }
-        return new LifecycleTransaction(tracker, transactionLogs.getType(), readers);
+        return new LifecycleTransaction(tracker, transactionLog.getType(), readers);
     }
 
     /**
@@ -524,27 +526,34 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
 
     public void trackNew(SSTable table)
     {
-        transactionLogs.trackNew(table);
+        transactionLog.trackNew(table);
     }
 
     public void untrackNew(SSTable table)
     {
-        transactionLogs.untrackNew(table);
+        transactionLog.untrackNew(table);
     }
 
     public static void removeUnfinishedLeftovers(CFMetaData metadata)
     {
-        TransactionLogs.removeUnfinishedLeftovers(metadata);
+        TransactionLog.removeUnfinishedLeftovers(metadata);
     }
 
-    public static Set<File> getTemporaryFiles(CFMetaData metadata, File folder)
-    {
-        return TransactionLogs.getTemporaryFiles(metadata, folder);
-    }
-
-    public static Set<File> getLogFiles(CFMetaData metadata)
+    /**
+     * Get the files in the folder specified, provided that the filter returns true.
+     * A filter is given each file and its type, and decides which files should be returned
+     * and which should be discarded. To classify files into their type, we read transaction
+     * log files. Should we fail to read these log files after a few times, we look at onTxnErr
+     * to determine what to do.
+     *
+     * @param folder - the folder to scan
+     * @param onTxnErr - how to handle a failure to read a txn log file
+     * @param filter - A function that receives each file and its type, it should return true to have the file returned
+     * @return - the list of files that were scanned and for which the filter returned true
+     */
+    public static List<File> getFiles(Path folder, BiFunction<File, Directories.FileType, Boolean> filter, Directories.OnTxnErr onTxnErr)
     {
-        return TransactionLogs.getLogFiles(metadata);
+        return new TransactionLog.FileLister(folder, filter, onTxnErr).list();
     }
 
     // a class representing the current state of the reader within this transaction, encoding the actions both logged

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index ea76fd6..6f6aca9 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -227,7 +227,7 @@ public class Tracker
      */
     public Throwable dropSSTables(final Predicate<SSTableReader> remove, OperationType operationType, Throwable accumulate)
     {
-        try (TransactionLogs txnLogs = new TransactionLogs(operationType, cfstore.metadata, this))
+        try (TransactionLog txnLogs = new TransactionLog(operationType, cfstore.metadata, this))
         {
             Pair<View, View> result = apply(view -> {
                 Set<SSTableReader> toremove = copyOf(filter(view.sstables, and(remove, notIn(view.compacting))));
@@ -239,7 +239,7 @@ public class Tracker
 
             // It is important that any method accepting/returning a Throwable never throws an exception, and does its best
             // to complete the instructions given to it
-            List<TransactionLogs.Obsoletion> obsoletions = new ArrayList<>();
+            List<TransactionLog.Obsoletion> obsoletions = new ArrayList<>();
             accumulate = prepareForObsoletion(removed, txnLogs, obsoletions, accumulate);
             try
             {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java b/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
new file mode 100644
index 0000000..a9e460c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
@@ -0,0 +1,1137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.SecureDirectoryStream;
+import java.util.*;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Runnables;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.RefCounted;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+import static org.apache.cassandra.db.Directories.OnTxnErr;
+import static org.apache.cassandra.db.Directories.FileType;
+import static org.apache.cassandra.utils.Throwables.maybeFail;
+import static org.apache.cassandra.utils.Throwables.merge;
+
+/**
+ * IMPORTANT: When this object is involved in a transactional graph, and is not encapsulated in a LifecycleTransaction,
+ * for correct behaviour its commit MUST occur before any others, since it may legitimately fail. This is consistent
+ * with the Transactional API, which permits one failing action to occur at the beginning of the commit phase, but also
+ * *requires* that the prepareToCommit() phase only take actions that can be rolled back.
+ *
+ * A class that tracks sstable files involved in a transaction across sstables:
+ * if the transaction succeeds the old files should be deleted and the new ones kept; vice-versa if it fails.
+ *
+ * The transaction log file contains new and old sstables as follows:
+ *
+ * add:[sstable-2][CRC]
+ * remove:[sstable-1,max_update_time,num files][CRC]
+ *
+ * where sstable-2 is a new sstable to be retained if the transaction succeeds and sstable-1 is an old sstable to be
+ * removed. CRC is an incremental CRC of the file content up to this point. For old sstable files we also log the
+ * last update time of all files for the sstable descriptor and a checksum of vital properties such as update times
+ * and file sizes.
+ *
+ * Upon commit we add a final line to the log file:
+ *
+ * commit:[commit_time][CRC]
+ *
+ * When the transaction log is cleaned-up by the TransactionTidier, which happens only after any old sstables have been
+ * osoleted, then any sstable files for old sstables are removed before deleting the transaction log if the transaction
+ * was committed, vice-versa if the transaction was aborted.
+ *
+ * On start-up we look for any transaction log files and repeat the cleanup process described above.
+ *
+ * See CASSANDRA-7066 for full details.
+ */
+public class TransactionLog extends Transactional.AbstractTransactional implements Transactional
+{
+    private static final Logger logger = LoggerFactory.getLogger(TransactionLog.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, TimeUnit.HOURS);
+
+    /**
+     * If the format of the lines in the transaction log is wrong or the checksum
+     * does not match, then we throw this exception.
+     */
+    public static final class CorruptTransactionLogException extends RuntimeException
+    {
+        public final TransactionFile file;
+
+        public CorruptTransactionLogException(String message, TransactionFile file)
+        {
+            super(message);
+            this.file = file;
+        }
+    }
+
+    public enum RecordType
+    {
+        ADD,    // new files to be retained on commit
+        REMOVE, // old files to be retained on abort
+        COMMIT, // commit flag
+        ABORT;  // abort flag
+        public static RecordType fromPrefix(String prefix)
+        {
+            return valueOf(prefix.toUpperCase());
+        }
+    }
+
+    /**
+     * A log file record, each record is encoded in one line and has different
+     * content depending on the record type.
+     */
+    final static class Record
+    {
+        public final RecordType type;
+        public final String relativeFilePath;
+        public final long updateTime;
+        public final int numFiles;
+        public final String record;
+
+        static String REGEX_STR = "^(add|remove|commit|abort):\\[([^,]*),?([^,]*),?([^,]*)\\]$";
+        static Pattern REGEX = Pattern.compile(REGEX_STR, Pattern.CASE_INSENSITIVE); // (add|remove|commit|abort):[*,*,*]
+
+        public static Record make(String record, boolean isLast)
+        {
+            try
+            {
+                Matcher matcher = REGEX.matcher(record);
+                if (!matcher.matches() || matcher.groupCount() != 4)
+                    throw new IllegalStateException(String.format("Invalid record \"%s\"", record));
+
+                RecordType type = RecordType.fromPrefix(matcher.group(1));
+                return new Record(type, matcher.group(2), Long.valueOf(matcher.group(3)), Integer.valueOf(matcher.group(4)), record);
+            }
+            catch (Throwable t)
+            {
+                if (!isLast)
+                    throw t;
+
+                int pos = record.indexOf(':');
+                if (pos <= 0)
+                    throw t;
+
+                RecordType recordType;
+                try
+                {
+                    recordType = RecordType.fromPrefix(record.substring(0, pos));
+                }
+                catch (Throwable ignore)
+                {
+                    throw t;
+                }
+
+                return new Record(recordType, "", 0, 0, record);
+
+            }
+        }
+
+        public static Record makeCommit(long updateTime)
+        {
+            return new Record(RecordType.COMMIT, "", updateTime, 0, "");
+        }
+
+        public static Record makeAbort(long updateTime)
+        {
+            return new Record(RecordType.ABORT, "", updateTime, 0, "");
+        }
+
+        public static Record makeNew(String relativeFilePath)
+        {
+            return new Record(RecordType.ADD, relativeFilePath, 0, 0, "");
+        }
+
+        public static Record makeOld(String parentFolder, String relativeFilePath)
+        {
+            return makeOld(getTrackedFiles(parentFolder, relativeFilePath), relativeFilePath);
+        }
+
+        public static Record makeOld(List<File> files, String relativeFilePath)
+        {
+            long lastModified = files.stream()
+                                     .mapToLong(File::lastModified)
+                                     .reduce(0L, Long::max);
+            return new Record(RecordType.REMOVE, relativeFilePath, lastModified, files.size(), "");
+        }
+
+        private Record(RecordType type,
+                       String relativeFilePath,
+                       long updateTime,
+                       int numFiles,
+                       String record)
+        {
+            this.type = type;
+            this.relativeFilePath = hasFilePath(type) ? relativeFilePath : ""; // only meaningful for some records
+            this.updateTime = type == RecordType.REMOVE ? updateTime : 0; // only meaningful for old records
+            this.numFiles = type == RecordType.REMOVE ? numFiles : 0; // only meaningful for old records
+            this.record = record.isEmpty() ? format() : record;
+        }
+
+        private static boolean hasFilePath(RecordType type)
+        {
+            return type == RecordType.ADD || type == RecordType.REMOVE;
+        }
+
+        private String format()
+        {
+            return String.format("%s:[%s,%d,%d]", type.toString(), relativeFilePath, updateTime, numFiles);
+        }
+
+        public byte[] getBytes()
+        {
+            return record.getBytes();
+        }
+
+        public boolean verify(String parentFolder, boolean lastRecordIsCorrupt)
+        {
+            if (type != RecordType.REMOVE)
+                return true;
+
+            List<File> files = getTrackedFiles(parentFolder);
+
+            // Paranoid sanity checks: we create another record by looking at the files as they are
+            // on disk right now and make sure the information still matches
+            Record currentRecord = Record.makeOld(files, relativeFilePath);
+            if (updateTime != currentRecord.updateTime)
+            {
+                logger.error("Possible disk corruption detected for sstable [{}], record [{}]: last update time [{}] should have been [{}]",
+                             relativeFilePath,
+                             record,
+                             new Date(currentRecord.updateTime),
+                             new Date(updateTime));
+                return false;
+            }
+
+            if (lastRecordIsCorrupt && currentRecord.numFiles < numFiles)
+            { // if we found a corruption in the last record, then we continue only if the number of files matches exactly.
+                logger.error("Possible disk corruption detected for sstable [{}], record [{}]: number of files [{}] should have been [{}]",
+                             relativeFilePath,
+                             record,
+                             currentRecord.numFiles,
+                             numFiles);
+                return false;
+            }
+
+            return true;
+        }
+
+        public List<File> getTrackedFiles(String parentFolder)
+        {
+            if (!hasFilePath(type))
+                return Collections.emptyList();
+
+            return getTrackedFiles(parentFolder, relativeFilePath);
+        }
+
+        public static List<File> getTrackedFiles(String parentFolder, String relativeFilePath)
+        {
+            return Arrays.asList(new File(parentFolder).listFiles((dir, name) -> name.startsWith(relativeFilePath)));
+        }
+
+        @Override
+        public int hashCode()
+        {
+            // see comment in equals
+            return Objects.hash(type, relativeFilePath);
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (obj == null)
+                return false;
+
+            if (getClass() != obj.getClass())
+                return false;
+
+            final Record other = (Record)obj;
+
+            // we exclude on purpose checksum, update time and count as
+            // we don't want duplicated records that differ only by
+            // properties that might change on disk, especially COMMIT records,
+            // there should be only one regardless of update time
+            return type.equals(other.type) &&
+                   relativeFilePath.equals(other.relativeFilePath);
+        }
+
+        @Override
+        public String toString()
+        {
+            return record;
+        }
+    }
+
+    /**
+     * The transaction log file, which contains many records.
+     */
+    final static class TransactionFile
+    {
+        static String EXT = ".log";
+        static char SEP = '_';
+        // cc_txn_opname_id.log (where cc is one of the sstable versions defined in BigVersion)
+        static String FILE_REGEX_STR = String.format("^(.{2})_txn_(.*)_(.*)%s$", EXT);
+        static Pattern FILE_REGEX = Pattern.compile(FILE_REGEX_STR);
+        static String LINE_REGEX_STR = "^(.*)\\[(\\d*)\\]$"; // *[checksum]
+        static Pattern LINE_REGEX = Pattern.compile(LINE_REGEX_STR);
+
+        public final File file;
+        public final TransactionData parent;
+        public final Set<Record> records = new HashSet<>();
+        public final Checksum checksum = new CRC32();
+
+        public TransactionFile(TransactionData parent)
+        {
+            this.file = new File(parent.getFileName());
+            this.parent = parent;
+        }
+
+        public void readRecords()
+        {
+            records.clear();
+            checksum.reset();
+
+            Iterator<String> it = FileUtils.readLines(file).iterator();
+            while(it.hasNext())
+                records.add(readRecord(it.next(), !it.hasNext())); // JLS execution order is left-to-right
+
+            for (Record record : records)
+            {
+                if (!record.verify(parent.getFolder(), false))
+                    throw new CorruptTransactionLogException(String.format("Failed to verify transaction %s record [%s]: possible disk corruption, aborting", parent.getId(), record),
+                                                             this);
+            }
+        }
+
+        private Record readRecord(String line, boolean isLast)
+        {
+            Matcher matcher = LINE_REGEX.matcher(line);
+            if (!matcher.matches() || matcher.groupCount() != 2)
+            {
+                handleReadRecordError(String.format("cannot parse line \"%s\"", line), isLast);
+                return Record.make(line, isLast);
+            }
+
+            byte[] bytes = matcher.group(1).getBytes();
+            checksum.update(bytes, 0, bytes.length);
+
+            if (checksum.getValue() != Long.valueOf(matcher.group(2)))
+                handleReadRecordError(String.format("invalid line checksum %s for \"%s\"", matcher.group(2), line), isLast);
+
+            try
+            {
+                return Record.make(matcher.group(1), isLast);
+            }
+            catch (Throwable t)
+            {
+                throw new CorruptTransactionLogException(String.format("Cannot make record \"%s\": %s", line, t.getMessage()), this);
+            }
+        }
+
+        private void handleReadRecordError(String message, boolean isLast)
+        {
+            if (isLast)
+            {
+                for (Record record : records)
+                {
+                    if (!record.verify(parent.getFolder(), true))
+                        throw new CorruptTransactionLogException(String.format("Last record of transaction %s is corrupt [%s] and at least " +
+                                                                               "one previous record does not match state on disk, possible disk corruption, aborting",
+                                                                               parent.getId(), message),
+                                                                 this);
+                }
+
+                // if only the last record is corrupt and all other records have matching files on disk, @see verifyRecord,
+                // then we simply exited whilst serializing the last record and we carry on
+                logger.warn(String.format("Last record of transaction %s is corrupt or incomplete [%s], but all previous records match state on disk; continuing", parent.getId(), message));
+
+            }
+            else
+            {
+                throw new CorruptTransactionLogException(String.format("Non-last record of transaction %s is corrupt [%s], possible disk corruption, aborting", parent.getId(), message), this);
+            }
+        }
+
+        public void commit()
+        {
+            assert !completed() : "Already completed!";
+            addRecord(Record.makeCommit(System.currentTimeMillis()));
+        }
+
+        public void abort()
+        {
+            assert !completed() : "Already completed!";
+            addRecord(Record.makeAbort(System.currentTimeMillis()));
+        }
+
+        public boolean committed()
+        {
+            return records.contains(Record.makeCommit(0));
+        }
+
+        public boolean aborted()
+        {
+            return records.contains(Record.makeAbort(0));
+        }
+
+        public boolean completed()
+        {
+            return committed() || aborted();
+        }
+
+        public boolean add(RecordType type, SSTable table)
+        {
+            Record record = makeRecord(type, table);
+            if (records.contains(record))
+                return false;
+
+            addRecord(record);
+            return true;
+        }
+
+        private Record makeRecord(RecordType type, SSTable table)
+        {
+            String relativePath = FileUtils.getRelativePath(parent.getFolder(), table.descriptor.baseFilename());
+            if (type == RecordType.ADD)
+            {
+                return Record.makeNew(relativePath);
+            }
+            else if (type == RecordType.REMOVE)
+            {
+                return Record.makeOld(parent.getFolder(), relativePath);
+            }
+            else
+            {
+                throw new AssertionError("Invalid record type " + type);
+            }
+        }
+
+        private void addRecord(Record record)
+        {
+            // we only checksum the records, not the checksums themselves
+            byte[] bytes = record.getBytes();
+            checksum.update(bytes, 0, bytes.length);
+
+            records.add(record);
+            FileUtils.append(file, String.format("%s[%d]", record, checksum.getValue()));
+
+            parent.sync();
+        }
+
+        public void remove(RecordType type, SSTable table)
+        {
+            Record record = makeRecord(type, table);
+
+            assert records.contains(record) : String.format("[%s] is not tracked by %s", record, file);
+
+            records.remove(record);
+            deleteRecord(record);
+        }
+
+        public boolean contains(RecordType type, SSTable table)
+        {
+            return records.contains(makeRecord(type, table));
+        }
+
+        public void deleteRecords(RecordType type)
+        {
+            assert file.exists() : String.format("Expected %s to exists", file);
+            records.stream()
+                   .filter((r) -> r.type == type)
+                   .forEach(this::deleteRecord);
+            records.clear();
+        }
+
+        private void deleteRecord(Record record)
+        {
+            List<File> files = record.getTrackedFiles(parent.getFolder());
+            if (files.isEmpty())
+                return; // Files no longer exist, nothing to do
+
+            // we sort the files in ascending update time order so that the last update time
+            // stays the same even if we only partially delete files
+            files.sort((f1, f2) -> Long.compare(f1.lastModified(), f2.lastModified()));
+
+            files.forEach(TransactionLog::delete);
+        }
+
+        public Set<File> getTrackedFiles(RecordType type)
+        {
+            return records.stream()
+                          .filter((r) -> r.type == type)
+                          .map((r) -> r.getTrackedFiles(parent.getFolder()))
+                          .flatMap(List::stream)
+                          .collect(Collectors.toSet());
+        }
+
+        public void delete()
+        {
+            TransactionLog.delete(file);
+        }
+
+        public boolean exists()
+        {
+            return file.exists();
+        }
+
+        @Override
+        public String toString()
+        {
+            return FileUtils.getRelativePath(parent.getFolder(), FileUtils.getCanonicalPath(file));
+        }
+    }
+
+    /**
+     * We split the transaction data from TransactionLog that implements the behavior
+     * because we need to reconstruct any left-overs and clean them up, as well as work
+     * out which files are temporary. So for these cases we don't want the full
+     * transactional behavior, plus it's handy for the TransactionTidier.
+     */
+    final static class TransactionData implements AutoCloseable
+    {
+        private final OperationType opType;
+        private final UUID id;
+        private final File folder;
+        private final TransactionFile file;
+        private int folderDescriptor;
+
+        static TransactionData make(File logFile)
+        {
+            Matcher matcher = TransactionFile.FILE_REGEX.matcher(logFile.getName());
+            assert matcher.matches() && matcher.groupCount() == 3;
+
+            // For now we don't need this but it is there in case we need to change
+            // file format later on, the version is the sstable version as defined in BigFormat
+            //String version = matcher.group(1);
+
+            OperationType operationType = OperationType.fromFileName(matcher.group(2));
+            UUID id = UUID.fromString(matcher.group(3));
+
+            return new TransactionData(operationType, logFile.getParentFile(), id);
+        }
+
+        TransactionData(OperationType opType, File folder, UUID id)
+        {
+            this.opType = opType;
+            this.id = id;
+            this.folder = folder;
+            this.file = new TransactionFile(this);
+            this.folderDescriptor = CLibrary.tryOpenDirectory(folder.getPath());
+        }
+
+        public Throwable readLogFile(Throwable accumulate)
+        {
+            try
+            {
+                file.readRecords();
+            }
+            catch (Throwable t)
+            {
+                accumulate = merge(accumulate, t);
+            }
+
+            return accumulate;
+        }
+
+        public void close()
+        {
+            if (folderDescriptor > 0)
+            {
+                CLibrary.tryCloseFD(folderDescriptor);
+                folderDescriptor = -1;
+            }
+        }
+
+        void sync()
+        {
+            if (folderDescriptor > 0)
+                CLibrary.trySync(folderDescriptor);
+        }
+
+        OperationType getType()
+        {
+            return opType;
+        }
+
+        UUID getId()
+        {
+            return id;
+        }
+
+        boolean completed()
+        {
+            return  file.completed();
+        }
+
+        Throwable removeUnfinishedLeftovers(Throwable accumulate)
+        {
+            try
+            {
+                if (file.committed())
+                    file.deleteRecords(RecordType.REMOVE);
+                else
+                    file.deleteRecords(RecordType.ADD);
+
+                // we sync the parent file descriptor between contents and log deletion
+                // to ensure there is a happens before edge between them
+                sync();
+
+                file.delete();
+            }
+            catch (Throwable t)
+            {
+                accumulate = merge(accumulate, t);
+            }
+
+            return accumulate;
+        }
+
+        Set<File> getTemporaryFiles()
+        {
+            sync();
+
+            if (!file.exists())
+                return Collections.emptySet();
+
+            if (file.committed())
+                return file.getTrackedFiles(RecordType.REMOVE);
+            else
+                return file.getTrackedFiles(RecordType.ADD);
+        }
+
+        String getFileName()
+        {
+            String fileName = StringUtils.join(BigFormat.latestVersion,
+                                               TransactionFile.SEP,
+                                               "txn",
+                                               TransactionFile.SEP,
+                                               opType.fileName,
+                                               TransactionFile.SEP,
+                                               id.toString(),
+                                               TransactionFile.EXT);
+            return StringUtils.join(folder, File.separator, fileName);
+        }
+
+        String getFolder()
+        {
+            return folder.getPath();
+        }
+
+        static boolean isLogFile(String name)
+        {
+            return TransactionFile.FILE_REGEX.matcher(name).matches();
+        }
+
+        @VisibleForTesting
+        TransactionFile getLogFile()
+        {
+            return file;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("[%s]", file.toString());
+        }
+    }
+
+    private final Tracker tracker;
+    private final TransactionData data;
+    private final Ref<TransactionLog> selfRef;
+    // Deleting sstables is tricky because the mmapping might not have been finalized yet,
+    // and delete will fail (on Windows) until it is (we only force the unmapping on SUN VMs).
+    // Additionally, we need to make sure to delete the data file first, so on restart the others
+    // will be recognized as GCable.
+    private static final Queue<Runnable> failedDeletions = new ConcurrentLinkedQueue<>();
+
+    TransactionLog(OperationType opType, CFMetaData metadata)
+    {
+        this(opType, metadata, null);
+    }
+
+    TransactionLog(OperationType opType, CFMetaData metadata, Tracker tracker)
+    {
+        this(opType, new Directories(metadata), tracker);
+    }
+
+    TransactionLog(OperationType opType, Directories directories, Tracker tracker)
+    {
+        this(opType, directories.getDirectoryForNewSSTables(), tracker);
+    }
+
+    TransactionLog(OperationType opType, File folder, Tracker tracker)
+    {
+        this.tracker = tracker;
+        this.data = new TransactionData(opType,
+                                        folder,
+                                        UUIDGen.getTimeUUID());
+        this.selfRef = new Ref<>(this, new TransactionTidier(data));
+
+        if (logger.isDebugEnabled())
+            logger.debug("Created transaction logs with id {}", data.id);
+    }
+
+    /**
+     * Track a reader as new.
+     **/
+    void trackNew(SSTable table)
+    {
+        if (!data.file.add(RecordType.ADD, table))
+            throw new IllegalStateException(table + " is already tracked as new");
+    }
+
+    /**
+     * Stop tracking a reader as new.
+     */
+    void untrackNew(SSTable table)
+    {
+        data.file.remove(RecordType.ADD, table);
+    }
+
+    /**
+     * Schedule a reader for deletion as soon as it is fully unreferenced and the transaction
+     * has been committed.
+     */
+    SSTableTidier obsoleted(SSTableReader reader)
+    {
+        if (data.file.contains(RecordType.ADD, reader))
+        {
+            if (data.file.contains(RecordType.REMOVE, reader))
+                throw new IllegalArgumentException();
+
+            return new SSTableTidier(reader, true, this);
+        }
+
+        if (!data.file.add(RecordType.REMOVE, reader))
+            throw new IllegalStateException();
+
+        if (tracker != null)
+            tracker.notifyDeleting(reader);
+
+        return new SSTableTidier(reader, false, this);
+    }
+
+    OperationType getType()
+    {
+        return data.getType();
+    }
+
+    UUID getId()
+    {
+        return data.getId();
+    }
+
+    @VisibleForTesting
+    String getDataFolder()
+    {
+        return data.getFolder();
+    }
+
+    @VisibleForTesting
+    TransactionData getData()
+    {
+        return data;
+    }
+
+    private static void delete(File file)
+    {
+        try
+        {
+            if (logger.isDebugEnabled())
+                logger.debug("Deleting {}", file);
+
+            Files.delete(file.toPath());
+        }
+        catch (NoSuchFileException e)
+        {
+            logger.error("Unable to delete {} as it does not exist", file);
+        }
+        catch (IOException e)
+        {
+            logger.error("Unable to delete {}", file, e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * The transaction tidier.
+     *
+     * When the transaction reference is fully released we try to delete all the obsolete files
+     * depending on the transaction result, as well as the transaction log file.
+     */
+    private static class TransactionTidier implements RefCounted.Tidy, Runnable
+    {
+        private final TransactionData data;
+
+        public TransactionTidier(TransactionData data)
+        {
+            this.data = data;
+        }
+
+        public void tidy() throws Exception
+        {
+            run();
+        }
+
+        public String name()
+        {
+            return data.toString();
+        }
+
+        public void run()
+        {
+            if (logger.isDebugEnabled())
+                logger.debug("Removing files for transaction {}", name());
+
+            assert data.completed() : "Expected a completed transaction: " + data;
+
+            Throwable err = data.removeUnfinishedLeftovers(null);
+
+            if (err != null)
+            {
+                logger.info("Failed deleting files for transaction {}, we'll retry after GC and on on server restart", name(), err);
+                failedDeletions.add(this);
+            }
+            else
+            {
+                if (logger.isDebugEnabled())
+                    logger.debug("Closing file transaction {}", name());
+                data.close();
+            }
+        }
+    }
+
+    static class Obsoletion
+    {
+        final SSTableReader reader;
+        final SSTableTidier tidier;
+
+        public Obsoletion(SSTableReader reader, SSTableTidier tidier)
+        {
+            this.reader = reader;
+            this.tidier = tidier;
+        }
+    }
+
+    /**
+     * The SSTableReader tidier. When a reader is fully released and no longer referenced
+     * by any one, we run this. It keeps a reference to the parent transaction and releases
+     * it when done, so that the final transaction cleanup can run when all obsolete readers
+     * are released.
+     */
+    public static class SSTableTidier implements Runnable
+    {
+        // must not retain a reference to the SSTableReader, else leak detection cannot kick in
+        private final Descriptor desc;
+        private final long sizeOnDisk;
+        private final Tracker tracker;
+        private final boolean wasNew;
+        private final Ref<TransactionLog> parentRef;
+
+        public SSTableTidier(SSTableReader referent, boolean wasNew, TransactionLog parent)
+        {
+            this.desc = referent.descriptor;
+            this.sizeOnDisk = referent.bytesOnDisk();
+            this.tracker = parent.tracker;
+            this.wasNew = wasNew;
+            this.parentRef = parent.selfRef.tryRef();
+        }
+
+        public void run()
+        {
+            SystemKeyspace.clearSSTableReadMeter(desc.ksname, desc.cfname, desc.generation);
+
+            try
+            {
+                // If we can't successfully delete the DATA component, set the task to be retried later: see TransactionTidier
+                File datafile = new File(desc.filenameFor(Component.DATA));
+
+                delete(datafile);
+                // let the remainder be cleaned up by delete
+                SSTable.delete(desc, SSTable.discoverComponentsFor(desc));
+            }
+            catch (Throwable t)
+            {
+                logger.error("Failed deletion for {}, we'll retry after GC and on server restart", desc);
+                failedDeletions.add(this);
+                return;
+            }
+
+            if (tracker != null && tracker.cfstore != null && !wasNew)
+                tracker.cfstore.metric.totalDiskSpaceUsed.dec(sizeOnDisk);
+
+            // release the referent to the parent so that the all transaction files can be released
+            parentRef.release();
+        }
+
+        public void abort()
+        {
+            parentRef.release();
+        }
+    }
+
+    /**
+     * Retry all deletions that failed the first time around (presumably b/c the sstable was still mmap'd.)
+     * Useful because there are times when we know GC has been invoked; also exposed as an mbean.
+     */
+    public static void rescheduleFailedDeletions()
+    {
+        Runnable task;
+        while ( null != (task = failedDeletions.poll()))
+            ScheduledExecutors.nonPeriodicTasks.submit(task);
+    }
+
+    /**
+     * Deletions run on the nonPeriodicTasks executor, (both failedDeletions or global tidiers in SSTableReader)
+     * so by scheduling a new empty task and waiting for it we ensure any prior deletion has completed.
+     */
+    public static void waitForDeletions()
+    {
+        FBUtilities.waitOnFuture(ScheduledExecutors.nonPeriodicTasks.schedule(Runnables.doNothing(), 0, TimeUnit.MILLISECONDS));
+    }
+
+    @VisibleForTesting
+    Throwable complete(Throwable accumulate)
+    {
+        try
+        {
+            accumulate = selfRef.ensureReleased(accumulate);
+            return accumulate;
+        }
+        catch (Throwable t)
+        {
+            logger.error("Failed to complete file transaction {}", getId(), t);
+            return Throwables.merge(accumulate, t);
+        }
+    }
+
+    protected Throwable doCommit(Throwable accumulate)
+    {
+        data.file.commit();
+        return complete(accumulate);
+    }
+
+    protected Throwable doAbort(Throwable accumulate)
+    {
+        data.file.abort();
+        return complete(accumulate);
+    }
+
+    protected void doPrepare() { }
+
+    /**
+     * Called on startup to scan existing folders for any unfinished leftovers of
+     * operations that were ongoing when the process exited. Also called by the standalone
+     * sstableutil tool when the cleanup option is specified, @see StandaloneSSTableUtil.
+     *
+     */
+    static void removeUnfinishedLeftovers(CFMetaData metadata)
+    {
+        Throwable accumulate = null;
+
+        for (File dir : new Directories(metadata).getCFDirectories())
+        {
+            File[] logs = dir.listFiles((dir1, name) -> TransactionData.isLogFile(name));
+
+            for (File log : logs)
+            {
+                try (TransactionData data = TransactionData.make(log))
+                {
+                    accumulate = data.readLogFile(accumulate);
+                    if (accumulate == null)
+                        accumulate = data.removeUnfinishedLeftovers(accumulate);
+                    else
+                        logger.error("Possible disk corruption: failed to read transaction log {}", log, accumulate);
+                }
+            }
+        }
+
+        if (accumulate != null)
+            logger.error("Failed to remove unfinished transaction leftovers", accumulate);
+    }
+
+    /**
+     * A class for listing files in a folder. If we fail we try a few more times
+     * in case we are reading txn log files that are still being mutated.
+     */
+    static final class FileLister
+    {
+        // The maximum number of attempts for scanning the folder
+        private static final int MAX_ATTEMPTS = 5;
+
+        // The delay between each attempt
+        private static final int REATTEMPT_DELAY_MILLIS = 5;
+
+        // The folder to scan
+        private final Path folder;
+
+        // The filter determines which files the client wants returned, we pass to the filter
+        // the file and its type
+        private final BiFunction<File, FileType, Boolean> filter;
+
+        // This determines the behavior when we fail to read a txn log file after a few times (MAX_ATTEMPTS)
+        private final OnTxnErr onTxnErr;
+
+        // Each time we scan the folder we increment this counter, we scan at most for MAX_ATTEMPTS
+        private int attempts;
+
+        public FileLister(Path folder, BiFunction<File, FileType, Boolean> filter, OnTxnErr onTxnErr)
+        {
+            this.folder = folder;
+            this.filter = filter;
+            this.onTxnErr = onTxnErr;
+            this.attempts = 0;
+        }
+
+        public List<File> list()
+        {
+            while(true)
+            {
+                try
+                {
+                    return attemptList();
+                }
+                catch (Throwable t)
+                {
+                    if (attempts >= MAX_ATTEMPTS)
+                        throw new RuntimeException(String.format("Failed to list files in %s after multiple attempts, giving up", folder), t);
+
+                    logger.warn("Failed to list files in {} : {}", folder, t.getMessage());
+                    try
+                    {
+                        Thread.sleep(REATTEMPT_DELAY_MILLIS);
+                    }
+                    catch (InterruptedException e)
+                    {
+                        logger.error("Interrupted whilst waiting to reattempt listing files in {}, giving up", folder, e);
+                        throw new RuntimeException(String.format("Failed to list files in %s due to interruption, giving up", folder), t);
+                    }
+                }
+            }
+        }
+
+        List<File> attemptList() throws IOException
+        {
+            attempts++;
+
+            Map<File, FileType> files = new HashMap<>();
+            try (DirectoryStream<Path> in = Files.newDirectoryStream(folder))
+            {
+                if (!(in instanceof SecureDirectoryStream))
+                    noSpamLogger.error("This platform does not support atomic directory streams (SecureDirectoryStream); " +
+                                       "race conditions when loading sstable files could occurr");
+
+                in.forEach(path ->
+                           {
+                               File file = path.toFile();
+                               if (file.isDirectory())
+                                   return;
+
+                               if (TransactionData.isLogFile(file.getName()))
+                               {
+                                   Set<File> tmpFiles = getTemporaryFiles(file);
+                                   if (tmpFiles != null)
+                                   { // process the txn log file only if we can read it (tmpFiles != null)
+                                       tmpFiles.stream().forEach((f) -> files.put(f, FileType.TEMPORARY));
+                                       files.put(file, FileType.TXN_LOG);
+                                   }
+                               }
+                               else
+                               {
+                                   files.putIfAbsent(file, FileType.FINAL);
+                               }
+                           });
+            }
+
+            return files.entrySet().stream()
+                        .filter((e) -> filter.apply(e.getKey(), e.getValue()))
+                        .map(Map.Entry::getKey)
+                        .collect(Collectors.toList());
+        }
+
+        Set<File> getTemporaryFiles(File file)
+        {
+            try (TransactionData txn = TransactionData.make(file))
+            {
+                maybeFail(txn.readLogFile(null));
+                return txn.getTemporaryFiles();
+            }
+            catch(Throwable t)
+            {
+                // We always fail if the onTxnErr is set to THROW or if we haven't
+                // reached the maximum number of attempts yet. Otherwise
+                // we just log an error and continue as if the txn log file does not exist
+                // clients can choose which behavior they want via onTxnLogError
+                if (attempts < MAX_ATTEMPTS ||
+                    onTxnErr == OnTxnErr.THROW)
+                    throw new RuntimeException(t);
+
+                logger.error("Failed to read temporary files of txn log {}", file, t);
+                return null; // txn.getTemporaryFiles() could be empty so we must use null to differentiate
+            }
+        }
+    }
+
+    @VisibleForTesting
+    static Set<File> getTemporaryFiles(CFMetaData metadata, File folder)
+    {
+        Set<File> ret = new HashSet<>();
+
+        List<File> directories = new Directories(metadata).getCFDirectories();
+        directories.add(folder);
+        for (File dir : directories)
+            ret.addAll(new FileLister(dir.toPath(),
+                                      (file, type) -> type != FileType.FINAL,
+                                      OnTxnErr.IGNORE).list());
+
+        return ret;
+    }
+}


[6/7] cassandra git commit: Improve transaction log under FS corruption

Posted by be...@apache.org.
Improve transaction log under FS corruption

To address concerns with recovery from file system
failures, we consolidate the new and old log
files into a single file, introduce checksums to the
file, and record on-disk sstable state to permit
validation of the log file on restart. This permits
us to be pessimistic in the case of any failure,
leaving all affected files on disk.

patch by stefania; reviewed by benedict for CASSANDRA-7066


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/5726625a
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/5726625a
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/5726625a

Branch: refs/heads/trunk
Commit: 5726625a5c7cf47a67509540f7146d05b668bc20
Parents: 47b66d7
Author: Stefania Alborghetti <st...@datastax.com>
Authored: Wed Jul 29 12:01:01 2015 +0800
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Mon Aug 17 10:20:01 2015 +0100

----------------------------------------------------------------------
 NEWS.txt                                        |   18 +
 bin/sstablelister                               |   55 -
 bin/sstablelister.bat                           |   41 -
 bin/sstableutil                                 |   55 +
 bin/sstableutil.bat                             |   41 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |   12 +-
 .../org/apache/cassandra/db/Directories.java    |  145 ++-
 .../db/compaction/CompactionManagerMBean.java   |    3 +-
 .../cassandra/db/compaction/Upgrader.java       |    9 -
 .../writers/MajorLeveledCompactionWriter.java   |    9 +-
 .../apache/cassandra/db/lifecycle/Helpers.java  |   12 +-
 .../db/lifecycle/LifecycleTransaction.java      |   69 +-
 .../apache/cassandra/db/lifecycle/Tracker.java  |    4 +-
 .../cassandra/db/lifecycle/TransactionLog.java  | 1137 ++++++++++++++++++
 .../cassandra/db/lifecycle/TransactionLogs.java |  786 ------------
 .../cassandra/io/sstable/SSTableLoader.java     |  162 ++-
 .../io/sstable/format/SSTableReader.java        |   22 +-
 .../cassandra/io/sstable/format/Version.java    |    2 +
 .../io/sstable/format/big/BigFormat.java        |   12 +
 .../io/sstable/metadata/CompactionMetadata.java |   43 +-
 .../metadata/LegacyMetadataSerializer.java      |   13 +-
 .../io/sstable/metadata/MetadataCollector.java  |   28 +-
 .../org/apache/cassandra/io/util/FileUtils.java |    5 -
 .../apache/cassandra/service/GCInspector.java   |    4 +-
 .../apache/cassandra/service/StartupChecks.java |    3 +-
 .../cassandra/service/StorageService.java       |    4 +-
 .../cassandra/tools/SSTableExpiredBlockers.java |    3 +-
 .../cassandra/tools/SSTableLevelResetter.java   |    3 +-
 .../cassandra/tools/SSTableMetadataViewer.java  |    2 -
 .../cassandra/tools/SSTableOfflineRelevel.java  |    2 +-
 .../cassandra/tools/StandaloneLister.java       |  214 ----
 .../cassandra/tools/StandaloneSSTableUtil.java  |  241 ++++
 .../cassandra/tools/StandaloneScrubber.java     |    6 +-
 .../cassandra/tools/StandaloneSplitter.java     |    4 +-
 .../cassandra/tools/StandaloneUpgrader.java     |    6 +-
 .../cassandra/tools/StandaloneVerifier.java     |    2 +-
 .../apache/cassandra/tools/nodetool/Stop.java   |    2 +-
 .../cassandra/db/ColumnFamilyStoreTest.java     |    2 +-
 .../apache/cassandra/db/DirectoriesTest.java    |    7 +-
 .../org/apache/cassandra/db/KeyCacheTest.java   |    6 +-
 .../unit/org/apache/cassandra/db/ScrubTest.java |    4 +-
 .../cassandra/db/lifecycle/HelpersTest.java     |    9 +-
 .../db/lifecycle/LifecycleTransactionTest.java  |    2 +-
 .../db/lifecycle/RealTransactionsTest.java      |   15 +-
 .../cassandra/db/lifecycle/TrackerTest.java     |   90 +-
 .../db/lifecycle/TransactionLogTest.java        |  791 ++++++++++++
 .../db/lifecycle/TransactionLogsTest.java       |  581 ---------
 .../io/sstable/CQLSSTableWriterClientTest.java  |    7 -
 .../io/sstable/CQLSSTableWriterTest.java        |    1 -
 .../io/sstable/SSTableRewriterTest.java         |   36 +-
 .../metadata/MetadataSerializerTest.java        |    7 -
 .../org/apache/cassandra/schema/DefsTest.java   |   11 +-
 52 files changed, 2625 insertions(+), 2123 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/NEWS.txt
----------------------------------------------------------------------
diff --git a/NEWS.txt b/NEWS.txt
index ef764ad..365ed31 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -38,6 +38,24 @@ Upgrading
    - The default JVM flag -XX:+PerfDisableSharedMem will cause the following tools JVM
      to stop working: jps, jstack, jinfo, jmc, jcmd as well as 3rd party tools like Jolokia.
      If you wish to use these tools you can comment this flag out in cassandra-env.{sh,ps1}
+   - New transaction log files have been introduced to replace the compactions_in_progress
+     system table, temporary file markers (tmp and tmplink) and sstable ancerstors.
+     Therefore, compaction metadata no longer contains ancestors. Transaction log files
+     list sstable descriptors involved in compactions and other operations such as flushing
+     and streaming. Use the sstableutil tool to list any sstable files currently involved
+     in operations not yet completed, which previously would have been marked as temporary.
+     A transaction log file contains one sstable per line, with the prefix "add:" or "remove:".
+     They also contain a special line "commit", only inserted at the end when the transaction
+     is committed. On startup we use these files to cleanup any partial transactions that were
+     in progress when the process exited. If the commit line is found, we keep new sstables
+     (those with the "add" prefix) and delete the old sstables (those with the "remove" prefix),
+     vice-versa if the commit line is missing. Should you lose or delete these log files,
+     both old and new sstable files will be kept as live files, which will result in duplicated
+     sstables. These files are protected by incremental checksums so you should not manually
+     edit them. When restoring a full backup or moving sstable files, you should clean-up
+     any left over transactions and their temporary files first. You can use this command:
+      ===> sstableutil -c ks table
+     See CASSANDRA-7066 for full details.
    - New write stages have been added for batchlog and materialized view mutations
      you can set their size in cassandra.yaml
    - User defined functions are now executed in a sandbox.

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/bin/sstablelister
----------------------------------------------------------------------
diff --git a/bin/sstablelister b/bin/sstablelister
deleted file mode 100755
index a79409d..0000000
--- a/bin/sstablelister
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
-    for include in /usr/share/cassandra/cassandra.in.sh \
-                   /usr/local/share/cassandra/cassandra.in.sh \
-                   /opt/cassandra/cassandra.in.sh \
-                   ~/.cassandra.in.sh \
-                   "`dirname "$0"`/cassandra.in.sh"; do
-        if [ -r "$include" ]; then
-            . "$include"
-            break
-        fi
-    done
-elif [ -r "$CASSANDRA_INCLUDE" ]; then
-    . "$CASSANDRA_INCLUDE"
-fi
-
-# Use JAVA_HOME if set, otherwise look for java in PATH
-if [ -x "$JAVA_HOME/bin/java" ]; then
-    JAVA="$JAVA_HOME/bin/java"
-else
-    JAVA="`which java`"
-fi
-
-if [ -z "$CLASSPATH" ]; then
-    echo "You must set the CLASSPATH var" >&2
-    exit 1
-fi
-
-if [ "x$MAX_HEAP_SIZE" = "x" ]; then
-    MAX_HEAP_SIZE="256M"
-fi
-
-"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
-        -Dcassandra.storagedir="$cassandra_storagedir" \
-        -Dlogback.configurationFile=logback-tools.xml \
-        org.apache.cassandra.tools.StandaloneLister "$@"
-
-# vi:ai sw=4 ts=4 tw=0 et

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/bin/sstablelister.bat
----------------------------------------------------------------------
diff --git a/bin/sstablelister.bat b/bin/sstablelister.bat
deleted file mode 100644
index cb50a08..0000000
--- a/bin/sstablelister.bat
+++ /dev/null
@@ -1,41 +0,0 @@
-@REM
-@REM  Licensed to the Apache Software Foundation (ASF) under one or more
-@REM  contributor license agreements.  See the NOTICE file distributed with
-@REM  this work for additional information regarding copyright ownership.
-@REM  The ASF licenses this file to You under the Apache License, Version 2.0
-@REM  (the "License"); you may not use this file except in compliance with
-@REM  the License.  You may obtain a copy of the License at
-@REM
-@REM      http://www.apache.org/licenses/LICENSE-2.0
-@REM
-@REM  Unless required by applicable law or agreed to in writing, software
-@REM  distributed under the License is distributed on an "AS IS" BASIS,
-@REM  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@REM  See the License for the specific language governing permissions and
-@REM  limitations under the License.
-
-@echo off
-if "%OS%" == "Windows_NT" setlocal
-
-pushd "%~dp0"
-call cassandra.in.bat
-
-if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.StandaloneLister
-if NOT DEFINED JAVA_HOME goto :err
-
-REM ***** JAVA options *****
-set JAVA_OPTS=^
- -Dlogback.configurationFile=logback-tools.xml
-
-set TOOLS_PARAMS=
-
-"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %*
-goto finally
-
-:err
-echo JAVA_HOME environment variable must be set!
-pause
-
-:finally
-
-ENDLOCAL

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/bin/sstableutil
----------------------------------------------------------------------
diff --git a/bin/sstableutil b/bin/sstableutil
new file mode 100755
index 0000000..2860729
--- /dev/null
+++ b/bin/sstableutil
@@ -0,0 +1,55 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+    for include in /usr/share/cassandra/cassandra.in.sh \
+                   /usr/local/share/cassandra/cassandra.in.sh \
+                   /opt/cassandra/cassandra.in.sh \
+                   ~/.cassandra.in.sh \
+                   "`dirname "$0"`/cassandra.in.sh"; do
+        if [ -r "$include" ]; then
+            . "$include"
+            break
+        fi
+    done
+elif [ -r "$CASSANDRA_INCLUDE" ]; then
+    . "$CASSANDRA_INCLUDE"
+fi
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -x "$JAVA_HOME/bin/java" ]; then
+    JAVA="$JAVA_HOME/bin/java"
+else
+    JAVA="`which java`"
+fi
+
+if [ -z "$CLASSPATH" ]; then
+    echo "You must set the CLASSPATH var" >&2
+    exit 1
+fi
+
+if [ "x$MAX_HEAP_SIZE" = "x" ]; then
+    MAX_HEAP_SIZE="256M"
+fi
+
+"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
+        -Dcassandra.storagedir="$cassandra_storagedir" \
+        -Dlogback.configurationFile=logback-tools.xml \
+        org.apache.cassandra.tools.StandaloneSSTableUtil "$@"
+
+# vi:ai sw=4 ts=4 tw=0 et

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/bin/sstableutil.bat
----------------------------------------------------------------------
diff --git a/bin/sstableutil.bat b/bin/sstableutil.bat
new file mode 100644
index 0000000..bc3eb8a
--- /dev/null
+++ b/bin/sstableutil.bat
@@ -0,0 +1,41 @@
+@REM
+@REM  Licensed to the Apache Software Foundation (ASF) under one or more
+@REM  contributor license agreements.  See the NOTICE file distributed with
+@REM  this work for additional information regarding copyright ownership.
+@REM  The ASF licenses this file to You under the Apache License, Version 2.0
+@REM  (the "License"); you may not use this file except in compliance with
+@REM  the License.  You may obtain a copy of the License at
+@REM
+@REM      http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM  Unless required by applicable law or agreed to in writing, software
+@REM  distributed under the License is distributed on an "AS IS" BASIS,
+@REM  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM  See the License for the specific language governing permissions and
+@REM  limitations under the License.
+
+@echo off
+if "%OS%" == "Windows_NT" setlocal
+
+pushd "%~dp0"
+call cassandra.in.bat
+
+if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.StandaloneSSTableUtil
+if NOT DEFINED JAVA_HOME goto :err
+
+REM ***** JAVA options *****
+set JAVA_OPTS=^
+ -Dlogback.configurationFile=logback-tools.xml
+
+set TOOLS_PARAMS=
+
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %*
+goto finally
+
+:err
+echo JAVA_HOME environment variable must be set!
+pause
+
+:finally
+
+ENDLOCAL

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index c7d12a2..202047b 100644
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@ -353,7 +353,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
 
         if (data.loadsstables)
         {
-            Directories.SSTableLister sstableFiles = directories.sstableLister().skipTemporary(true);
+            Directories.SSTableLister sstableFiles = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
             Collection<SSTableReader> sstables = SSTableReader.openAll(sstableFiles.list().entrySet(), metadata);
             data.addInitialSSTables(sstables);
         }
@@ -459,7 +459,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         SystemKeyspace.removeTruncationRecord(metadata.cfId);
 
         data.dropSSTables();
-        TransactionLogs.waitForDeletions();
+        TransactionLog.waitForDeletions();
 
         indexManager.invalidate();
         materializedViewManager.invalidate();
@@ -503,7 +503,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
     {
         // get the max generation number, to prevent generation conflicts
         Directories directories = new Directories(metadata);
-        Directories.SSTableLister lister = directories.sstableLister().includeBackups(true);
+        Directories.SSTableLister lister = directories.sstableLister(Directories.OnTxnErr.IGNORE).includeBackups(true);
         List<Integer> generations = new ArrayList<Integer>();
         for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
         {
@@ -534,7 +534,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         LifecycleTransaction.removeUnfinishedLeftovers(metadata);
 
         logger.debug("Further extra check for orphan sstable files for {}", metadata.cfName);
-        for (Map.Entry<Descriptor,Set<Component>> sstableFiles : directories.sstableLister().list().entrySet())
+        for (Map.Entry<Descriptor,Set<Component>> sstableFiles : directories.sstableLister(Directories.OnTxnErr.IGNORE).list().entrySet())
         {
             Descriptor desc = sstableFiles.getKey();
             Set<Component> components = sstableFiles.getValue();
@@ -642,7 +642,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
             currentDescriptors.add(sstable.descriptor);
         Set<SSTableReader> newSSTables = new HashSet<>();
 
-        Directories.SSTableLister lister = directories.sstableLister().skipTemporary(true);
+        Directories.SSTableLister lister = directories.sstableLister(Directories.OnTxnErr.IGNORE).skipTemporary(true);
         for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet())
         {
             Descriptor descriptor = entry.getKey();
@@ -1644,7 +1644,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
         Map<Integer, SSTableReader> active = new HashMap<>();
         for (SSTableReader sstable : getSSTables(SSTableSet.CANONICAL))
             active.put(sstable.descriptor.generation, sstable);
-        Map<Descriptor, Set<Component>> snapshots = directories.sstableLister().snapshots(tag).list();
+        Map<Descriptor, Set<Component>> snapshots = directories.sstableLister(Directories.OnTxnErr.IGNORE).snapshots(tag).list();
         Refs<SSTableReader> refs = new Refs<>();
         try
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/Directories.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java
index bede4c4..fa01269 100644
--- a/src/java/org/apache/cassandra/db/Directories.java
+++ b/src/java/org/apache/cassandra/db/Directories.java
@@ -31,6 +31,7 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.util.*;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiFunction;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
@@ -92,7 +93,6 @@ public class Directories
 
     public static final String BACKUPS_SUBDIR = "backups";
     public static final String SNAPSHOT_SUBDIR = "snapshots";
-    public static final String TRANSACTIONS_SUBDIR = "transactions";
     public static final String SECONDARY_INDEX_NAME_SEPARATOR = ".";
 
     public static final DataDirectory[] dataDirectories;
@@ -142,7 +142,7 @@ public class Directories
     {
         X, W, XW, R, XR, RW, XRW;
 
-        private FileAction()
+        FileAction()
         {
         }
 
@@ -468,40 +468,6 @@ public class Directories
         }
     }
 
-    public static File getTransactionsDirectory(File folder)
-    {
-        return getOrCreate(folder, TRANSACTIONS_SUBDIR);
-    }
-
-    public List<File> getExistingDirectories(String subFolder)
-    {
-        List<File> ret = new ArrayList<>();
-        for (File dir : dataPaths)
-        {
-            File subDir = getExistingDirectory(dir, subFolder);
-            if (subDir != null)
-                ret.add(subDir);
-
-        }
-        return ret;
-    }
-
-    public static File getExistingDirectory(File folder, String subFolder)
-    {
-        File subDir = new File(folder, join(subFolder));
-        if (subDir.exists())
-        {
-            assert(subDir.isDirectory());
-            return subDir;
-        }
-        return null;
-    }
-
-    public SSTableLister sstableLister()
-    {
-        return new SSTableLister();
-    }
-
     public static class DataDirectory
     {
         public final File location;
@@ -549,10 +515,42 @@ public class Directories
         }
     }
 
+    /** The type of files that can be listed by SSTableLister, we never return txn logs,
+     * use LifecycleTransaction.getFiles() if you need txn logs. */
+    public enum FileType
+    {
+        /** A permanent sstable file that is safe to use. */
+        FINAL,
+
+        /** A temporary sstable file that will soon be deleted. */
+        TEMPORARY,
+
+        /** A transaction log file (contains information on final and temporary files). */
+        TXN_LOG
+    }
+
+    /**
+     * How to handle a failure to read a txn log file. Note that we will try a few
+     * times before giving up.
+     **/
+    public enum OnTxnErr
+    {
+        /** Throw the exception */
+        THROW,
+
+        /** Ignore the txn log file */
+        IGNORE
+    }
+
+    public SSTableLister sstableLister(OnTxnErr onTxnErr)
+    {
+        return new SSTableLister(onTxnErr);
+    }
+
     public class SSTableLister
     {
+        private final OnTxnErr onTxnErr;
         private boolean skipTemporary;
-        private boolean onlyTemporary;
         private boolean includeBackups;
         private boolean onlyBackups;
         private int nbFiles;
@@ -560,19 +558,16 @@ public class Directories
         private boolean filtered;
         private String snapshotName;
 
-        public SSTableLister skipTemporary(boolean b)
+        private SSTableLister(OnTxnErr onTxnErr)
         {
-            if (filtered)
-                throw new IllegalStateException("list() has already been called");
-            skipTemporary = b;
-            return this;
+            this.onTxnErr = onTxnErr;
         }
 
-        public SSTableLister onlyTemporary(boolean b)
+        public SSTableLister skipTemporary(boolean b)
         {
             if (filtered)
                 throw new IllegalStateException("list() has already been called");
-            onlyTemporary = b;
+            skipTemporary = b;
             return this;
         }
 
@@ -633,56 +628,54 @@ public class Directories
 
                 if (snapshotName != null)
                 {
-                    getSnapshotDirectory(location, snapshotName).listFiles(getFilter(location));
+                    LifecycleTransaction.getFiles(getSnapshotDirectory(location, snapshotName).toPath(), getFilter(), onTxnErr);
                     continue;
                 }
 
                 if (!onlyBackups)
-                    location.listFiles(getFilter(location));
+                    LifecycleTransaction.getFiles(location.toPath(), getFilter(), onTxnErr);
 
                 if (includeBackups)
-                    getBackupsDirectory(location).listFiles(getFilter(location));
+                    LifecycleTransaction.getFiles(getBackupsDirectory(location).toPath(), getFilter(), onTxnErr);
             }
+
             filtered = true;
         }
 
-        private FileFilter getFilter(File location)
+        private BiFunction<File, FileType, Boolean> getFilter()
         {
-           final Set<File> temporaryFiles = skipTemporary || onlyTemporary
-                                            ? LifecycleTransaction.getTemporaryFiles(metadata, location)
-                                            : Collections.<File>emptySet();
-
-            return new FileFilter()
+            // This function always return false since it adds to the components map
+            return (file, type) ->
             {
-                // This function always return false since accepts adds to the components map
-                public boolean accept(File file)
+                switch (type)
                 {
-                    if (file.isDirectory())
-                        return false;
-
-                    Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
-                    if (pair == null)
+                    case TXN_LOG:
                         return false;
+                    case TEMPORARY:
+                        if (skipTemporary)
+                            return false;
 
-                    // we are only interested in the SSTable files that belong to the specific ColumnFamily
-                    if (!pair.left.ksname.equals(metadata.ksName) || !pair.left.cfname.equals(metadata.cfName))
-                        return false;
+                    case FINAL:
+                        Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
+                        if (pair == null)
+                            return false;
 
-                    if (skipTemporary && temporaryFiles.contains(file))
-                        return false;
+                        // we are only interested in the SSTable files that belong to the specific ColumnFamily
+                        if (!pair.left.ksname.equals(metadata.ksName) || !pair.left.cfname.equals(metadata.cfName))
+                            return false;
 
-                    if (onlyTemporary && !temporaryFiles.contains(file))
+                        Set<Component> previous = components.get(pair.left);
+                        if (previous == null)
+                        {
+                            previous = new HashSet<>();
+                            components.put(pair.left, previous);
+                        }
+                        previous.add(pair.right);
+                        nbFiles++;
                         return false;
 
-                    Set<Component> previous = components.get(pair.left);
-                    if (previous == null)
-                    {
-                        previous = new HashSet<>();
-                        components.put(pair.left, previous);
-                    }
-                    previous.add(pair.right);
-                    nbFiles++;
-                    return false;
+                    default:
+                        throw new AssertionError();
                 }
             };
         }
@@ -917,7 +910,7 @@ public class Directories
         {
             super();
             Builder<String> builder = ImmutableSet.builder();
-            for (File file : sstableLister().listFiles())
+            for (File file : sstableLister(Directories.OnTxnErr.THROW).listFiles())
                 builder.add(file.getName());
             alive = builder.build();
         }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
index 8e200a1..d5da0fe 100644
--- a/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
+++ b/src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
@@ -58,7 +58,8 @@ public interface CompactionManagerMBean
     /**
      * Stop an individual running compaction using the compactionId.
      * @param compactionId Compaction ID of compaction to stop. Such IDs can be found in
-     *                     the compactions_in_progress table of the system keyspace.
+     *                     the transaction log files whose name starts with compaction_,
+     *                     located in the table transactions folder.
      */
     public void stopCompactionById(String compactionId);
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/Upgrader.java b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
index 5a36210..ebfd997 100644
--- a/src/java/org/apache/cassandra/db/compaction/Upgrader.java
+++ b/src/java/org/apache/cassandra/db/compaction/Upgrader.java
@@ -69,15 +69,6 @@ public class Upgrader
     private SSTableWriter createCompactionWriter(long repairedAt)
     {
         MetadataCollector sstableMetadataCollector = new MetadataCollector(cfs.getComparator());
-
-        // Get the max timestamp of the precompacted sstables
-        // and adds generation of live ancestors
-        sstableMetadataCollector.addAncestor(sstable.descriptor.generation);
-        for (Integer i : sstable.getAncestors())
-        {
-            if (new File(sstable.descriptor.withGeneration(i).filenameFor(Component.DATA)).exists())
-                sstableMetadataCollector.addAncestor(i);
-        }
         sstableMetadataCollector.sstableLevel(sstable.getSSTableLevel());
         return SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(directory)),
                                     estimatedRows,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
index a826809..73ce216 100644
--- a/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
+++ b/src/java/org/apache/cassandra/db/compaction/writers/MajorLeveledCompactionWriter.java
@@ -45,7 +45,6 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
     private long partitionsWritten = 0;
     private long totalWrittenInLevel = 0;
     private int sstablesWritten = 0;
-    private final boolean skipAncestors;
 
     public MajorLeveledCompactionWriter(ColumnFamilyStore cfs,
                                         LifecycleTransaction txn,
@@ -70,17 +69,13 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
         long estimatedSSTables = Math.max(1, SSTableReader.getTotalBytes(nonExpiredSSTables) / maxSSTableSize);
         long keysPerSSTable = estimatedTotalKeys / estimatedSSTables;
         File sstableDirectory = cfs.directories.getLocationForDisk(getWriteDirectory(expectedWriteSize));
-        skipAncestors = estimatedSSTables * allSSTables.size() > 200000; // magic number, avoid storing too much ancestor information since allSSTables are ancestors to *all* resulting sstables
-
-        if (skipAncestors)
-            logger.warn("Many sstables involved in compaction, skipping storing ancestor information to avoid running out of memory");
 
         @SuppressWarnings("resource")
         SSTableWriter writer = SSTableWriter.create(Descriptor.fromFilename(cfs.getSSTablePath(sstableDirectory)),
                                                     keysPerSSTable,
                                                     minRepairedAt,
                                                     cfs.metadata,
-                                                    new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel, skipAncestors),
+                                                    new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel),
                                                     SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
                                                     txn);
         sstableWriter.switchWriter(writer);
@@ -108,7 +103,7 @@ public class MajorLeveledCompactionWriter extends CompactionAwareWriter
                                                         averageEstimatedKeysPerSSTable,
                                                         minRepairedAt,
                                                         cfs.metadata,
-                                                        new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel, skipAncestors),
+                                                        new MetadataCollector(allSSTables, cfs.metadata.comparator, currentLevel),
                                                         SerializationHeader.make(cfs.metadata, nonExpiredSSTables),
                                                         txn);
             sstableWriter.switchWriter(writer);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
index 8cb92b9..98983c5 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Helpers.java
@@ -106,12 +106,12 @@ class Helpers
             assert !reader.isReplaced();
     }
 
-    static Throwable markObsolete(List<TransactionLogs.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable markObsolete(List<TransactionLog.Obsoletion> obsoletions, Throwable accumulate)
     {
         if (obsoletions == null || obsoletions.isEmpty())
             return accumulate;
 
-        for (TransactionLogs.Obsoletion obsoletion : obsoletions)
+        for (TransactionLog.Obsoletion obsoletion : obsoletions)
         {
             try
             {
@@ -125,13 +125,13 @@ class Helpers
         return accumulate;
     }
 
-    static Throwable prepareForObsoletion(Iterable<SSTableReader> readers, TransactionLogs txnLogs, List<TransactionLogs.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable prepareForObsoletion(Iterable<SSTableReader> readers, TransactionLog txnLogs, List<TransactionLog.Obsoletion> obsoletions, Throwable accumulate)
     {
         for (SSTableReader reader : readers)
         {
             try
             {
-                obsoletions.add(new TransactionLogs.Obsoletion(reader, txnLogs.obsoleted(reader)));
+                obsoletions.add(new TransactionLog.Obsoletion(reader, txnLogs.obsoleted(reader)));
             }
             catch (Throwable t)
             {
@@ -141,12 +141,12 @@ class Helpers
         return accumulate;
     }
 
-    static Throwable abortObsoletion(List<TransactionLogs.Obsoletion> obsoletions, Throwable accumulate)
+    static Throwable abortObsoletion(List<TransactionLog.Obsoletion> obsoletions, Throwable accumulate)
     {
         if (obsoletions == null || obsoletions.isEmpty())
             return accumulate;
 
-        for (TransactionLogs.Obsoletion obsoletion : obsoletions)
+        for (TransactionLog.Obsoletion obsoletion : obsoletions)
         {
             try
             {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
index 5bda4d4..c6cb979 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/LifecycleTransaction.java
@@ -18,10 +18,11 @@
 package org.apache.cassandra.db.lifecycle;
 
 import java.io.File;
+import java.nio.file.Path;
 import java.util.*;
+import java.util.function.BiFunction;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
 import com.google.common.base.Predicate;
 import com.google.common.collect.*;
 
@@ -29,6 +30,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.compaction.OperationType;
 import org.apache.cassandra.io.sstable.SSTable;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -96,7 +98,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
 
     public final Tracker tracker;
     // The transaction logs keep track of new and old sstable files
-    private final TransactionLogs transactionLogs;
+    private final TransactionLog transactionLog;
     // the original readers this transaction was opened over, and that it guards
     // (no other transactions may operate over these readers concurrently)
     private final Set<SSTableReader> originals = new HashSet<>();
@@ -113,7 +115,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
     private final State staged = new State();
 
     // the tidier and their readers, to be used for marking readers obsoleted during a commit
-    private List<TransactionLogs.Obsoletion> obsoletions;
+    private List<TransactionLog.Obsoletion> obsoletions;
 
     /**
      * construct a Transaction for use in an offline operation
@@ -141,7 +143,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
     public static LifecycleTransaction offline(OperationType operationType, CFMetaData metadata)
     {
         Tracker dummy = new Tracker(null, false);
-        return new LifecycleTransaction(dummy, new TransactionLogs(operationType, metadata, dummy), Collections.emptyList());
+        return new LifecycleTransaction(dummy, new TransactionLog(operationType, metadata, dummy), Collections.emptyList());
     }
 
     /**
@@ -150,18 +152,18 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
     public static LifecycleTransaction offline(OperationType operationType, File operationFolder)
     {
         Tracker dummy = new Tracker(null, false);
-        return new LifecycleTransaction(dummy, new TransactionLogs(operationType, operationFolder, dummy), Collections.emptyList());
+        return new LifecycleTransaction(dummy, new TransactionLog(operationType, operationFolder, dummy), Collections.emptyList());
     }
 
     LifecycleTransaction(Tracker tracker, OperationType operationType, Iterable<SSTableReader> readers)
     {
-        this(tracker, new TransactionLogs(operationType, getMetadata(tracker, readers), tracker), readers);
+        this(tracker, new TransactionLog(operationType, getMetadata(tracker, readers), tracker), readers);
     }
 
-    LifecycleTransaction(Tracker tracker, TransactionLogs transactionLogs, Iterable<SSTableReader> readers)
+    LifecycleTransaction(Tracker tracker, TransactionLog transactionLog, Iterable<SSTableReader> readers)
     {
         this.tracker = tracker;
-        this.transactionLogs = transactionLogs;
+        this.transactionLog = transactionLog;
         for (SSTableReader reader : readers)
         {
             originals.add(reader);
@@ -185,19 +187,19 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
         return null;
     }
 
-    public TransactionLogs logs()
+    public TransactionLog log()
     {
-        return transactionLogs;
+        return transactionLog;
     }
 
     public OperationType opType()
     {
-        return transactionLogs.getType();
+        return transactionLog.getType();
     }
 
     public UUID opId()
     {
-        return transactionLogs.getId();
+        return transactionLog.getId();
     }
 
     public void doPrepare()
@@ -210,8 +212,8 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
 
         // prepare for compaction obsolete readers as long as they were part of the original set
         // since those that are not original are early readers that share the same desc with the finals
-        maybeFail(prepareForObsoletion(filterIn(logged.obsolete, originals), transactionLogs, obsoletions = new ArrayList<>(), null));
-        transactionLogs.prepareToCommit();
+        maybeFail(prepareForObsoletion(filterIn(logged.obsolete, originals), transactionLog, obsoletions = new ArrayList<>(), null));
+        transactionLog.prepareToCommit();
     }
 
     /**
@@ -226,7 +228,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
         maybeFail(accumulate);
 
         // transaction log commit failure means we must abort; safe commit is not possible
-        maybeFail(transactionLogs.commit(null));
+        maybeFail(transactionLog.commit(null));
 
         // this is now the point of no return; we cannot safely rollback, so we ignore exceptions until we're done
         // we restore state by obsoleting our obsolete files, releasing our references to them, and updating our size
@@ -235,7 +237,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
         accumulate = markObsolete(obsoletions, accumulate);
         accumulate = tracker.updateSizeTracking(logged.obsolete, logged.update, accumulate);
         accumulate = release(selfRefs(logged.obsolete), accumulate);
-        accumulate = tracker.notifySSTablesChanged(originals, logged.update, transactionLogs.getType(), accumulate);
+        accumulate = tracker.notifySSTablesChanged(originals, logged.update, transactionLog.getType(), accumulate);
 
         return accumulate;
     }
@@ -251,16 +253,16 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
         accumulate = abortObsoletion(obsoletions, accumulate);
 
         if (logged.isEmpty() && staged.isEmpty())
-            return transactionLogs.abort(accumulate);
+            return transactionLog.abort(accumulate);
 
         // mark obsolete all readers that are not versions of those present in the original set
         Iterable<SSTableReader> obsolete = filterOut(concatUniq(staged.update, logged.update), originals);
         logger.debug("Obsoleting {}", obsolete);
 
-        accumulate = prepareForObsoletion(obsolete, transactionLogs, obsoletions = new ArrayList<>(), accumulate);
+        accumulate = prepareForObsoletion(obsolete, transactionLog, obsoletions = new ArrayList<>(), accumulate);
         // it's safe to abort even if committed, see maybeFail in doCommit() above, in this case it will just report
         // a failure to abort, which is useful information to have for debug
-        accumulate = transactionLogs.abort(accumulate);
+        accumulate = transactionLog.abort(accumulate);
         accumulate = markObsolete(obsoletions, accumulate);
 
         // replace all updated readers with a version restored to its original state
@@ -491,7 +493,7 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
             originals.remove(reader);
             marked.remove(reader);
         }
-        return new LifecycleTransaction(tracker, transactionLogs.getType(), readers);
+        return new LifecycleTransaction(tracker, transactionLog.getType(), readers);
     }
 
     /**
@@ -524,27 +526,34 @@ public class LifecycleTransaction extends Transactional.AbstractTransactional
 
     public void trackNew(SSTable table)
     {
-        transactionLogs.trackNew(table);
+        transactionLog.trackNew(table);
     }
 
     public void untrackNew(SSTable table)
     {
-        transactionLogs.untrackNew(table);
+        transactionLog.untrackNew(table);
     }
 
     public static void removeUnfinishedLeftovers(CFMetaData metadata)
     {
-        TransactionLogs.removeUnfinishedLeftovers(metadata);
+        TransactionLog.removeUnfinishedLeftovers(metadata);
     }
 
-    public static Set<File> getTemporaryFiles(CFMetaData metadata, File folder)
-    {
-        return TransactionLogs.getTemporaryFiles(metadata, folder);
-    }
-
-    public static Set<File> getLogFiles(CFMetaData metadata)
+    /**
+     * Get the files in the folder specified, provided that the filter returns true.
+     * A filter is given each file and its type, and decides which files should be returned
+     * and which should be discarded. To classify files into their type, we read transaction
+     * log files. Should we fail to read these log files after a few times, we look at onTxnErr
+     * to determine what to do.
+     *
+     * @param folder - the folder to scan
+     * @param onTxnErr - how to handle a failure to read a txn log file
+     * @param filter - A function that receives each file and its type, it should return true to have the file returned
+     * @return - the list of files that were scanned and for which the filter returned true
+     */
+    public static List<File> getFiles(Path folder, BiFunction<File, Directories.FileType, Boolean> filter, Directories.OnTxnErr onTxnErr)
     {
-        return TransactionLogs.getLogFiles(metadata);
+        return new TransactionLog.FileLister(folder, filter, onTxnErr).list();
     }
 
     // a class representing the current state of the reader within this transaction, encoding the actions both logged

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
index ea76fd6..6f6aca9 100644
--- a/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
+++ b/src/java/org/apache/cassandra/db/lifecycle/Tracker.java
@@ -227,7 +227,7 @@ public class Tracker
      */
     public Throwable dropSSTables(final Predicate<SSTableReader> remove, OperationType operationType, Throwable accumulate)
     {
-        try (TransactionLogs txnLogs = new TransactionLogs(operationType, cfstore.metadata, this))
+        try (TransactionLog txnLogs = new TransactionLog(operationType, cfstore.metadata, this))
         {
             Pair<View, View> result = apply(view -> {
                 Set<SSTableReader> toremove = copyOf(filter(view.sstables, and(remove, notIn(view.compacting))));
@@ -239,7 +239,7 @@ public class Tracker
 
             // It is important that any method accepting/returning a Throwable never throws an exception, and does its best
             // to complete the instructions given to it
-            List<TransactionLogs.Obsoletion> obsoletions = new ArrayList<>();
+            List<TransactionLog.Obsoletion> obsoletions = new ArrayList<>();
             accumulate = prepareForObsoletion(removed, txnLogs, obsoletions, accumulate);
             try
             {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java b/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
new file mode 100644
index 0000000..a9e460c
--- /dev/null
+++ b/src/java/org/apache/cassandra/db/lifecycle/TransactionLog.java
@@ -0,0 +1,1137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.db.lifecycle;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.SecureDirectoryStream;
+import java.util.*;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.Runnables;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.SystemKeyspace;
+import org.apache.cassandra.db.compaction.OperationType;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTable;
+import org.apache.cassandra.io.sstable.format.SSTableReader;
+import org.apache.cassandra.io.sstable.format.big.BigFormat;
+import org.apache.cassandra.io.util.FileUtils;
+import org.apache.cassandra.utils.*;
+import org.apache.cassandra.utils.concurrent.Ref;
+import org.apache.cassandra.utils.concurrent.RefCounted;
+import org.apache.cassandra.utils.concurrent.Transactional;
+
+import static org.apache.cassandra.db.Directories.OnTxnErr;
+import static org.apache.cassandra.db.Directories.FileType;
+import static org.apache.cassandra.utils.Throwables.maybeFail;
+import static org.apache.cassandra.utils.Throwables.merge;
+
+/**
+ * IMPORTANT: When this object is involved in a transactional graph, and is not encapsulated in a LifecycleTransaction,
+ * for correct behaviour its commit MUST occur before any others, since it may legitimately fail. This is consistent
+ * with the Transactional API, which permits one failing action to occur at the beginning of the commit phase, but also
+ * *requires* that the prepareToCommit() phase only take actions that can be rolled back.
+ *
+ * A class that tracks sstable files involved in a transaction across sstables:
+ * if the transaction succeeds the old files should be deleted and the new ones kept; vice-versa if it fails.
+ *
+ * The transaction log file contains new and old sstables as follows:
+ *
+ * add:[sstable-2][CRC]
+ * remove:[sstable-1,max_update_time,num files][CRC]
+ *
+ * where sstable-2 is a new sstable to be retained if the transaction succeeds and sstable-1 is an old sstable to be
+ * removed. CRC is an incremental CRC of the file content up to this point. For old sstable files we also log the
+ * last update time of all files for the sstable descriptor and a checksum of vital properties such as update times
+ * and file sizes.
+ *
+ * Upon commit we add a final line to the log file:
+ *
+ * commit:[commit_time][CRC]
+ *
+ * When the transaction log is cleaned-up by the TransactionTidier, which happens only after any old sstables have been
+ * osoleted, then any sstable files for old sstables are removed before deleting the transaction log if the transaction
+ * was committed, vice-versa if the transaction was aborted.
+ *
+ * On start-up we look for any transaction log files and repeat the cleanup process described above.
+ *
+ * See CASSANDRA-7066 for full details.
+ */
+public class TransactionLog extends Transactional.AbstractTransactional implements Transactional
+{
+    private static final Logger logger = LoggerFactory.getLogger(TransactionLog.class);
+    private static final NoSpamLogger noSpamLogger = NoSpamLogger.getLogger(logger, 1L, TimeUnit.HOURS);
+
+    /**
+     * If the format of the lines in the transaction log is wrong or the checksum
+     * does not match, then we throw this exception.
+     */
+    public static final class CorruptTransactionLogException extends RuntimeException
+    {
+        public final TransactionFile file;
+
+        public CorruptTransactionLogException(String message, TransactionFile file)
+        {
+            super(message);
+            this.file = file;
+        }
+    }
+
+    public enum RecordType
+    {
+        ADD,    // new files to be retained on commit
+        REMOVE, // old files to be retained on abort
+        COMMIT, // commit flag
+        ABORT;  // abort flag
+        public static RecordType fromPrefix(String prefix)
+        {
+            return valueOf(prefix.toUpperCase());
+        }
+    }
+
+    /**
+     * A log file record, each record is encoded in one line and has different
+     * content depending on the record type.
+     */
+    final static class Record
+    {
+        public final RecordType type;
+        public final String relativeFilePath;
+        public final long updateTime;
+        public final int numFiles;
+        public final String record;
+
+        static String REGEX_STR = "^(add|remove|commit|abort):\\[([^,]*),?([^,]*),?([^,]*)\\]$";
+        static Pattern REGEX = Pattern.compile(REGEX_STR, Pattern.CASE_INSENSITIVE); // (add|remove|commit|abort):[*,*,*]
+
+        public static Record make(String record, boolean isLast)
+        {
+            try
+            {
+                Matcher matcher = REGEX.matcher(record);
+                if (!matcher.matches() || matcher.groupCount() != 4)
+                    throw new IllegalStateException(String.format("Invalid record \"%s\"", record));
+
+                RecordType type = RecordType.fromPrefix(matcher.group(1));
+                return new Record(type, matcher.group(2), Long.valueOf(matcher.group(3)), Integer.valueOf(matcher.group(4)), record);
+            }
+            catch (Throwable t)
+            {
+                if (!isLast)
+                    throw t;
+
+                int pos = record.indexOf(':');
+                if (pos <= 0)
+                    throw t;
+
+                RecordType recordType;
+                try
+                {
+                    recordType = RecordType.fromPrefix(record.substring(0, pos));
+                }
+                catch (Throwable ignore)
+                {
+                    throw t;
+                }
+
+                return new Record(recordType, "", 0, 0, record);
+
+            }
+        }
+
+        public static Record makeCommit(long updateTime)
+        {
+            return new Record(RecordType.COMMIT, "", updateTime, 0, "");
+        }
+
+        public static Record makeAbort(long updateTime)
+        {
+            return new Record(RecordType.ABORT, "", updateTime, 0, "");
+        }
+
+        public static Record makeNew(String relativeFilePath)
+        {
+            return new Record(RecordType.ADD, relativeFilePath, 0, 0, "");
+        }
+
+        public static Record makeOld(String parentFolder, String relativeFilePath)
+        {
+            return makeOld(getTrackedFiles(parentFolder, relativeFilePath), relativeFilePath);
+        }
+
+        public static Record makeOld(List<File> files, String relativeFilePath)
+        {
+            long lastModified = files.stream()
+                                     .mapToLong(File::lastModified)
+                                     .reduce(0L, Long::max);
+            return new Record(RecordType.REMOVE, relativeFilePath, lastModified, files.size(), "");
+        }
+
+        private Record(RecordType type,
+                       String relativeFilePath,
+                       long updateTime,
+                       int numFiles,
+                       String record)
+        {
+            this.type = type;
+            this.relativeFilePath = hasFilePath(type) ? relativeFilePath : ""; // only meaningful for some records
+            this.updateTime = type == RecordType.REMOVE ? updateTime : 0; // only meaningful for old records
+            this.numFiles = type == RecordType.REMOVE ? numFiles : 0; // only meaningful for old records
+            this.record = record.isEmpty() ? format() : record;
+        }
+
+        private static boolean hasFilePath(RecordType type)
+        {
+            return type == RecordType.ADD || type == RecordType.REMOVE;
+        }
+
+        private String format()
+        {
+            return String.format("%s:[%s,%d,%d]", type.toString(), relativeFilePath, updateTime, numFiles);
+        }
+
+        public byte[] getBytes()
+        {
+            return record.getBytes();
+        }
+
+        public boolean verify(String parentFolder, boolean lastRecordIsCorrupt)
+        {
+            if (type != RecordType.REMOVE)
+                return true;
+
+            List<File> files = getTrackedFiles(parentFolder);
+
+            // Paranoid sanity checks: we create another record by looking at the files as they are
+            // on disk right now and make sure the information still matches
+            Record currentRecord = Record.makeOld(files, relativeFilePath);
+            if (updateTime != currentRecord.updateTime)
+            {
+                logger.error("Possible disk corruption detected for sstable [{}], record [{}]: last update time [{}] should have been [{}]",
+                             relativeFilePath,
+                             record,
+                             new Date(currentRecord.updateTime),
+                             new Date(updateTime));
+                return false;
+            }
+
+            if (lastRecordIsCorrupt && currentRecord.numFiles < numFiles)
+            { // if we found a corruption in the last record, then we continue only if the number of files matches exactly.
+                logger.error("Possible disk corruption detected for sstable [{}], record [{}]: number of files [{}] should have been [{}]",
+                             relativeFilePath,
+                             record,
+                             currentRecord.numFiles,
+                             numFiles);
+                return false;
+            }
+
+            return true;
+        }
+
+        public List<File> getTrackedFiles(String parentFolder)
+        {
+            if (!hasFilePath(type))
+                return Collections.emptyList();
+
+            return getTrackedFiles(parentFolder, relativeFilePath);
+        }
+
+        public static List<File> getTrackedFiles(String parentFolder, String relativeFilePath)
+        {
+            return Arrays.asList(new File(parentFolder).listFiles((dir, name) -> name.startsWith(relativeFilePath)));
+        }
+
+        @Override
+        public int hashCode()
+        {
+            // see comment in equals
+            return Objects.hash(type, relativeFilePath);
+        }
+
+        @Override
+        public boolean equals(Object obj)
+        {
+            if (obj == null)
+                return false;
+
+            if (getClass() != obj.getClass())
+                return false;
+
+            final Record other = (Record)obj;
+
+            // we exclude on purpose checksum, update time and count as
+            // we don't want duplicated records that differ only by
+            // properties that might change on disk, especially COMMIT records,
+            // there should be only one regardless of update time
+            return type.equals(other.type) &&
+                   relativeFilePath.equals(other.relativeFilePath);
+        }
+
+        @Override
+        public String toString()
+        {
+            return record;
+        }
+    }
+
+    /**
+     * The transaction log file, which contains many records.
+     */
+    final static class TransactionFile
+    {
+        static String EXT = ".log";
+        static char SEP = '_';
+        // cc_txn_opname_id.log (where cc is one of the sstable versions defined in BigVersion)
+        static String FILE_REGEX_STR = String.format("^(.{2})_txn_(.*)_(.*)%s$", EXT);
+        static Pattern FILE_REGEX = Pattern.compile(FILE_REGEX_STR);
+        static String LINE_REGEX_STR = "^(.*)\\[(\\d*)\\]$"; // *[checksum]
+        static Pattern LINE_REGEX = Pattern.compile(LINE_REGEX_STR);
+
+        public final File file;
+        public final TransactionData parent;
+        public final Set<Record> records = new HashSet<>();
+        public final Checksum checksum = new CRC32();
+
+        public TransactionFile(TransactionData parent)
+        {
+            this.file = new File(parent.getFileName());
+            this.parent = parent;
+        }
+
+        public void readRecords()
+        {
+            records.clear();
+            checksum.reset();
+
+            Iterator<String> it = FileUtils.readLines(file).iterator();
+            while(it.hasNext())
+                records.add(readRecord(it.next(), !it.hasNext())); // JLS execution order is left-to-right
+
+            for (Record record : records)
+            {
+                if (!record.verify(parent.getFolder(), false))
+                    throw new CorruptTransactionLogException(String.format("Failed to verify transaction %s record [%s]: possible disk corruption, aborting", parent.getId(), record),
+                                                             this);
+            }
+        }
+
+        private Record readRecord(String line, boolean isLast)
+        {
+            Matcher matcher = LINE_REGEX.matcher(line);
+            if (!matcher.matches() || matcher.groupCount() != 2)
+            {
+                handleReadRecordError(String.format("cannot parse line \"%s\"", line), isLast);
+                return Record.make(line, isLast);
+            }
+
+            byte[] bytes = matcher.group(1).getBytes();
+            checksum.update(bytes, 0, bytes.length);
+
+            if (checksum.getValue() != Long.valueOf(matcher.group(2)))
+                handleReadRecordError(String.format("invalid line checksum %s for \"%s\"", matcher.group(2), line), isLast);
+
+            try
+            {
+                return Record.make(matcher.group(1), isLast);
+            }
+            catch (Throwable t)
+            {
+                throw new CorruptTransactionLogException(String.format("Cannot make record \"%s\": %s", line, t.getMessage()), this);
+            }
+        }
+
+        private void handleReadRecordError(String message, boolean isLast)
+        {
+            if (isLast)
+            {
+                for (Record record : records)
+                {
+                    if (!record.verify(parent.getFolder(), true))
+                        throw new CorruptTransactionLogException(String.format("Last record of transaction %s is corrupt [%s] and at least " +
+                                                                               "one previous record does not match state on disk, possible disk corruption, aborting",
+                                                                               parent.getId(), message),
+                                                                 this);
+                }
+
+                // if only the last record is corrupt and all other records have matching files on disk, @see verifyRecord,
+                // then we simply exited whilst serializing the last record and we carry on
+                logger.warn(String.format("Last record of transaction %s is corrupt or incomplete [%s], but all previous records match state on disk; continuing", parent.getId(), message));
+
+            }
+            else
+            {
+                throw new CorruptTransactionLogException(String.format("Non-last record of transaction %s is corrupt [%s], possible disk corruption, aborting", parent.getId(), message), this);
+            }
+        }
+
+        public void commit()
+        {
+            assert !completed() : "Already completed!";
+            addRecord(Record.makeCommit(System.currentTimeMillis()));
+        }
+
+        public void abort()
+        {
+            assert !completed() : "Already completed!";
+            addRecord(Record.makeAbort(System.currentTimeMillis()));
+        }
+
+        public boolean committed()
+        {
+            return records.contains(Record.makeCommit(0));
+        }
+
+        public boolean aborted()
+        {
+            return records.contains(Record.makeAbort(0));
+        }
+
+        public boolean completed()
+        {
+            return committed() || aborted();
+        }
+
+        public boolean add(RecordType type, SSTable table)
+        {
+            Record record = makeRecord(type, table);
+            if (records.contains(record))
+                return false;
+
+            addRecord(record);
+            return true;
+        }
+
+        private Record makeRecord(RecordType type, SSTable table)
+        {
+            String relativePath = FileUtils.getRelativePath(parent.getFolder(), table.descriptor.baseFilename());
+            if (type == RecordType.ADD)
+            {
+                return Record.makeNew(relativePath);
+            }
+            else if (type == RecordType.REMOVE)
+            {
+                return Record.makeOld(parent.getFolder(), relativePath);
+            }
+            else
+            {
+                throw new AssertionError("Invalid record type " + type);
+            }
+        }
+
+        private void addRecord(Record record)
+        {
+            // we only checksum the records, not the checksums themselves
+            byte[] bytes = record.getBytes();
+            checksum.update(bytes, 0, bytes.length);
+
+            records.add(record);
+            FileUtils.append(file, String.format("%s[%d]", record, checksum.getValue()));
+
+            parent.sync();
+        }
+
+        public void remove(RecordType type, SSTable table)
+        {
+            Record record = makeRecord(type, table);
+
+            assert records.contains(record) : String.format("[%s] is not tracked by %s", record, file);
+
+            records.remove(record);
+            deleteRecord(record);
+        }
+
+        public boolean contains(RecordType type, SSTable table)
+        {
+            return records.contains(makeRecord(type, table));
+        }
+
+        public void deleteRecords(RecordType type)
+        {
+            assert file.exists() : String.format("Expected %s to exists", file);
+            records.stream()
+                   .filter((r) -> r.type == type)
+                   .forEach(this::deleteRecord);
+            records.clear();
+        }
+
+        private void deleteRecord(Record record)
+        {
+            List<File> files = record.getTrackedFiles(parent.getFolder());
+            if (files.isEmpty())
+                return; // Files no longer exist, nothing to do
+
+            // we sort the files in ascending update time order so that the last update time
+            // stays the same even if we only partially delete files
+            files.sort((f1, f2) -> Long.compare(f1.lastModified(), f2.lastModified()));
+
+            files.forEach(TransactionLog::delete);
+        }
+
+        public Set<File> getTrackedFiles(RecordType type)
+        {
+            return records.stream()
+                          .filter((r) -> r.type == type)
+                          .map((r) -> r.getTrackedFiles(parent.getFolder()))
+                          .flatMap(List::stream)
+                          .collect(Collectors.toSet());
+        }
+
+        public void delete()
+        {
+            TransactionLog.delete(file);
+        }
+
+        public boolean exists()
+        {
+            return file.exists();
+        }
+
+        @Override
+        public String toString()
+        {
+            return FileUtils.getRelativePath(parent.getFolder(), FileUtils.getCanonicalPath(file));
+        }
+    }
+
+    /**
+     * We split the transaction data from TransactionLog that implements the behavior
+     * because we need to reconstruct any left-overs and clean them up, as well as work
+     * out which files are temporary. So for these cases we don't want the full
+     * transactional behavior, plus it's handy for the TransactionTidier.
+     */
+    final static class TransactionData implements AutoCloseable
+    {
+        private final OperationType opType;
+        private final UUID id;
+        private final File folder;
+        private final TransactionFile file;
+        private int folderDescriptor;
+
+        static TransactionData make(File logFile)
+        {
+            Matcher matcher = TransactionFile.FILE_REGEX.matcher(logFile.getName());
+            assert matcher.matches() && matcher.groupCount() == 3;
+
+            // For now we don't need this but it is there in case we need to change
+            // file format later on, the version is the sstable version as defined in BigFormat
+            //String version = matcher.group(1);
+
+            OperationType operationType = OperationType.fromFileName(matcher.group(2));
+            UUID id = UUID.fromString(matcher.group(3));
+
+            return new TransactionData(operationType, logFile.getParentFile(), id);
+        }
+
+        TransactionData(OperationType opType, File folder, UUID id)
+        {
+            this.opType = opType;
+            this.id = id;
+            this.folder = folder;
+            this.file = new TransactionFile(this);
+            this.folderDescriptor = CLibrary.tryOpenDirectory(folder.getPath());
+        }
+
+        public Throwable readLogFile(Throwable accumulate)
+        {
+            try
+            {
+                file.readRecords();
+            }
+            catch (Throwable t)
+            {
+                accumulate = merge(accumulate, t);
+            }
+
+            return accumulate;
+        }
+
+        public void close()
+        {
+            if (folderDescriptor > 0)
+            {
+                CLibrary.tryCloseFD(folderDescriptor);
+                folderDescriptor = -1;
+            }
+        }
+
+        void sync()
+        {
+            if (folderDescriptor > 0)
+                CLibrary.trySync(folderDescriptor);
+        }
+
+        OperationType getType()
+        {
+            return opType;
+        }
+
+        UUID getId()
+        {
+            return id;
+        }
+
+        boolean completed()
+        {
+            return  file.completed();
+        }
+
+        Throwable removeUnfinishedLeftovers(Throwable accumulate)
+        {
+            try
+            {
+                if (file.committed())
+                    file.deleteRecords(RecordType.REMOVE);
+                else
+                    file.deleteRecords(RecordType.ADD);
+
+                // we sync the parent file descriptor between contents and log deletion
+                // to ensure there is a happens before edge between them
+                sync();
+
+                file.delete();
+            }
+            catch (Throwable t)
+            {
+                accumulate = merge(accumulate, t);
+            }
+
+            return accumulate;
+        }
+
+        Set<File> getTemporaryFiles()
+        {
+            sync();
+
+            if (!file.exists())
+                return Collections.emptySet();
+
+            if (file.committed())
+                return file.getTrackedFiles(RecordType.REMOVE);
+            else
+                return file.getTrackedFiles(RecordType.ADD);
+        }
+
+        String getFileName()
+        {
+            String fileName = StringUtils.join(BigFormat.latestVersion,
+                                               TransactionFile.SEP,
+                                               "txn",
+                                               TransactionFile.SEP,
+                                               opType.fileName,
+                                               TransactionFile.SEP,
+                                               id.toString(),
+                                               TransactionFile.EXT);
+            return StringUtils.join(folder, File.separator, fileName);
+        }
+
+        String getFolder()
+        {
+            return folder.getPath();
+        }
+
+        static boolean isLogFile(String name)
+        {
+            return TransactionFile.FILE_REGEX.matcher(name).matches();
+        }
+
+        @VisibleForTesting
+        TransactionFile getLogFile()
+        {
+            return file;
+        }
+
+        @Override
+        public String toString()
+        {
+            return String.format("[%s]", file.toString());
+        }
+    }
+
+    private final Tracker tracker;
+    private final TransactionData data;
+    private final Ref<TransactionLog> selfRef;
+    // Deleting sstables is tricky because the mmapping might not have been finalized yet,
+    // and delete will fail (on Windows) until it is (we only force the unmapping on SUN VMs).
+    // Additionally, we need to make sure to delete the data file first, so on restart the others
+    // will be recognized as GCable.
+    private static final Queue<Runnable> failedDeletions = new ConcurrentLinkedQueue<>();
+
+    TransactionLog(OperationType opType, CFMetaData metadata)
+    {
+        this(opType, metadata, null);
+    }
+
+    TransactionLog(OperationType opType, CFMetaData metadata, Tracker tracker)
+    {
+        this(opType, new Directories(metadata), tracker);
+    }
+
+    TransactionLog(OperationType opType, Directories directories, Tracker tracker)
+    {
+        this(opType, directories.getDirectoryForNewSSTables(), tracker);
+    }
+
+    TransactionLog(OperationType opType, File folder, Tracker tracker)
+    {
+        this.tracker = tracker;
+        this.data = new TransactionData(opType,
+                                        folder,
+                                        UUIDGen.getTimeUUID());
+        this.selfRef = new Ref<>(this, new TransactionTidier(data));
+
+        if (logger.isDebugEnabled())
+            logger.debug("Created transaction logs with id {}", data.id);
+    }
+
+    /**
+     * Track a reader as new.
+     **/
+    void trackNew(SSTable table)
+    {
+        if (!data.file.add(RecordType.ADD, table))
+            throw new IllegalStateException(table + " is already tracked as new");
+    }
+
+    /**
+     * Stop tracking a reader as new.
+     */
+    void untrackNew(SSTable table)
+    {
+        data.file.remove(RecordType.ADD, table);
+    }
+
+    /**
+     * Schedule a reader for deletion as soon as it is fully unreferenced and the transaction
+     * has been committed.
+     */
+    SSTableTidier obsoleted(SSTableReader reader)
+    {
+        if (data.file.contains(RecordType.ADD, reader))
+        {
+            if (data.file.contains(RecordType.REMOVE, reader))
+                throw new IllegalArgumentException();
+
+            return new SSTableTidier(reader, true, this);
+        }
+
+        if (!data.file.add(RecordType.REMOVE, reader))
+            throw new IllegalStateException();
+
+        if (tracker != null)
+            tracker.notifyDeleting(reader);
+
+        return new SSTableTidier(reader, false, this);
+    }
+
+    OperationType getType()
+    {
+        return data.getType();
+    }
+
+    UUID getId()
+    {
+        return data.getId();
+    }
+
+    @VisibleForTesting
+    String getDataFolder()
+    {
+        return data.getFolder();
+    }
+
+    @VisibleForTesting
+    TransactionData getData()
+    {
+        return data;
+    }
+
+    private static void delete(File file)
+    {
+        try
+        {
+            if (logger.isDebugEnabled())
+                logger.debug("Deleting {}", file);
+
+            Files.delete(file.toPath());
+        }
+        catch (NoSuchFileException e)
+        {
+            logger.error("Unable to delete {} as it does not exist", file);
+        }
+        catch (IOException e)
+        {
+            logger.error("Unable to delete {}", file, e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * The transaction tidier.
+     *
+     * When the transaction reference is fully released we try to delete all the obsolete files
+     * depending on the transaction result, as well as the transaction log file.
+     */
+    private static class TransactionTidier implements RefCounted.Tidy, Runnable
+    {
+        private final TransactionData data;
+
+        public TransactionTidier(TransactionData data)
+        {
+            this.data = data;
+        }
+
+        public void tidy() throws Exception
+        {
+            run();
+        }
+
+        public String name()
+        {
+            return data.toString();
+        }
+
+        public void run()
+        {
+            if (logger.isDebugEnabled())
+                logger.debug("Removing files for transaction {}", name());
+
+            assert data.completed() : "Expected a completed transaction: " + data;
+
+            Throwable err = data.removeUnfinishedLeftovers(null);
+
+            if (err != null)
+            {
+                logger.info("Failed deleting files for transaction {}, we'll retry after GC and on on server restart", name(), err);
+                failedDeletions.add(this);
+            }
+            else
+            {
+                if (logger.isDebugEnabled())
+                    logger.debug("Closing file transaction {}", name());
+                data.close();
+            }
+        }
+    }
+
+    static class Obsoletion
+    {
+        final SSTableReader reader;
+        final SSTableTidier tidier;
+
+        public Obsoletion(SSTableReader reader, SSTableTidier tidier)
+        {
+            this.reader = reader;
+            this.tidier = tidier;
+        }
+    }
+
+    /**
+     * The SSTableReader tidier. When a reader is fully released and no longer referenced
+     * by any one, we run this. It keeps a reference to the parent transaction and releases
+     * it when done, so that the final transaction cleanup can run when all obsolete readers
+     * are released.
+     */
+    public static class SSTableTidier implements Runnable
+    {
+        // must not retain a reference to the SSTableReader, else leak detection cannot kick in
+        private final Descriptor desc;
+        private final long sizeOnDisk;
+        private final Tracker tracker;
+        private final boolean wasNew;
+        private final Ref<TransactionLog> parentRef;
+
+        public SSTableTidier(SSTableReader referent, boolean wasNew, TransactionLog parent)
+        {
+            this.desc = referent.descriptor;
+            this.sizeOnDisk = referent.bytesOnDisk();
+            this.tracker = parent.tracker;
+            this.wasNew = wasNew;
+            this.parentRef = parent.selfRef.tryRef();
+        }
+
+        public void run()
+        {
+            SystemKeyspace.clearSSTableReadMeter(desc.ksname, desc.cfname, desc.generation);
+
+            try
+            {
+                // If we can't successfully delete the DATA component, set the task to be retried later: see TransactionTidier
+                File datafile = new File(desc.filenameFor(Component.DATA));
+
+                delete(datafile);
+                // let the remainder be cleaned up by delete
+                SSTable.delete(desc, SSTable.discoverComponentsFor(desc));
+            }
+            catch (Throwable t)
+            {
+                logger.error("Failed deletion for {}, we'll retry after GC and on server restart", desc);
+                failedDeletions.add(this);
+                return;
+            }
+
+            if (tracker != null && tracker.cfstore != null && !wasNew)
+                tracker.cfstore.metric.totalDiskSpaceUsed.dec(sizeOnDisk);
+
+            // release the referent to the parent so that the all transaction files can be released
+            parentRef.release();
+        }
+
+        public void abort()
+        {
+            parentRef.release();
+        }
+    }
+
+    /**
+     * Retry all deletions that failed the first time around (presumably b/c the sstable was still mmap'd.)
+     * Useful because there are times when we know GC has been invoked; also exposed as an mbean.
+     */
+    public static void rescheduleFailedDeletions()
+    {
+        Runnable task;
+        while ( null != (task = failedDeletions.poll()))
+            ScheduledExecutors.nonPeriodicTasks.submit(task);
+    }
+
+    /**
+     * Deletions run on the nonPeriodicTasks executor, (both failedDeletions or global tidiers in SSTableReader)
+     * so by scheduling a new empty task and waiting for it we ensure any prior deletion has completed.
+     */
+    public static void waitForDeletions()
+    {
+        FBUtilities.waitOnFuture(ScheduledExecutors.nonPeriodicTasks.schedule(Runnables.doNothing(), 0, TimeUnit.MILLISECONDS));
+    }
+
+    @VisibleForTesting
+    Throwable complete(Throwable accumulate)
+    {
+        try
+        {
+            accumulate = selfRef.ensureReleased(accumulate);
+            return accumulate;
+        }
+        catch (Throwable t)
+        {
+            logger.error("Failed to complete file transaction {}", getId(), t);
+            return Throwables.merge(accumulate, t);
+        }
+    }
+
+    protected Throwable doCommit(Throwable accumulate)
+    {
+        data.file.commit();
+        return complete(accumulate);
+    }
+
+    protected Throwable doAbort(Throwable accumulate)
+    {
+        data.file.abort();
+        return complete(accumulate);
+    }
+
+    protected void doPrepare() { }
+
+    /**
+     * Called on startup to scan existing folders for any unfinished leftovers of
+     * operations that were ongoing when the process exited. Also called by the standalone
+     * sstableutil tool when the cleanup option is specified, @see StandaloneSSTableUtil.
+     *
+     */
+    static void removeUnfinishedLeftovers(CFMetaData metadata)
+    {
+        Throwable accumulate = null;
+
+        for (File dir : new Directories(metadata).getCFDirectories())
+        {
+            File[] logs = dir.listFiles((dir1, name) -> TransactionData.isLogFile(name));
+
+            for (File log : logs)
+            {
+                try (TransactionData data = TransactionData.make(log))
+                {
+                    accumulate = data.readLogFile(accumulate);
+                    if (accumulate == null)
+                        accumulate = data.removeUnfinishedLeftovers(accumulate);
+                    else
+                        logger.error("Possible disk corruption: failed to read transaction log {}", log, accumulate);
+                }
+            }
+        }
+
+        if (accumulate != null)
+            logger.error("Failed to remove unfinished transaction leftovers", accumulate);
+    }
+
+    /**
+     * A class for listing files in a folder. If we fail we try a few more times
+     * in case we are reading txn log files that are still being mutated.
+     */
+    static final class FileLister
+    {
+        // The maximum number of attempts for scanning the folder
+        private static final int MAX_ATTEMPTS = 5;
+
+        // The delay between each attempt
+        private static final int REATTEMPT_DELAY_MILLIS = 5;
+
+        // The folder to scan
+        private final Path folder;
+
+        // The filter determines which files the client wants returned, we pass to the filter
+        // the file and its type
+        private final BiFunction<File, FileType, Boolean> filter;
+
+        // This determines the behavior when we fail to read a txn log file after a few times (MAX_ATTEMPTS)
+        private final OnTxnErr onTxnErr;
+
+        // Each time we scan the folder we increment this counter, we scan at most for MAX_ATTEMPTS
+        private int attempts;
+
+        public FileLister(Path folder, BiFunction<File, FileType, Boolean> filter, OnTxnErr onTxnErr)
+        {
+            this.folder = folder;
+            this.filter = filter;
+            this.onTxnErr = onTxnErr;
+            this.attempts = 0;
+        }
+
+        public List<File> list()
+        {
+            while(true)
+            {
+                try
+                {
+                    return attemptList();
+                }
+                catch (Throwable t)
+                {
+                    if (attempts >= MAX_ATTEMPTS)
+                        throw new RuntimeException(String.format("Failed to list files in %s after multiple attempts, giving up", folder), t);
+
+                    logger.warn("Failed to list files in {} : {}", folder, t.getMessage());
+                    try
+                    {
+                        Thread.sleep(REATTEMPT_DELAY_MILLIS);
+                    }
+                    catch (InterruptedException e)
+                    {
+                        logger.error("Interrupted whilst waiting to reattempt listing files in {}, giving up", folder, e);
+                        throw new RuntimeException(String.format("Failed to list files in %s due to interruption, giving up", folder), t);
+                    }
+                }
+            }
+        }
+
+        List<File> attemptList() throws IOException
+        {
+            attempts++;
+
+            Map<File, FileType> files = new HashMap<>();
+            try (DirectoryStream<Path> in = Files.newDirectoryStream(folder))
+            {
+                if (!(in instanceof SecureDirectoryStream))
+                    noSpamLogger.error("This platform does not support atomic directory streams (SecureDirectoryStream); " +
+                                       "race conditions when loading sstable files could occurr");
+
+                in.forEach(path ->
+                           {
+                               File file = path.toFile();
+                               if (file.isDirectory())
+                                   return;
+
+                               if (TransactionData.isLogFile(file.getName()))
+                               {
+                                   Set<File> tmpFiles = getTemporaryFiles(file);
+                                   if (tmpFiles != null)
+                                   { // process the txn log file only if we can read it (tmpFiles != null)
+                                       tmpFiles.stream().forEach((f) -> files.put(f, FileType.TEMPORARY));
+                                       files.put(file, FileType.TXN_LOG);
+                                   }
+                               }
+                               else
+                               {
+                                   files.putIfAbsent(file, FileType.FINAL);
+                               }
+                           });
+            }
+
+            return files.entrySet().stream()
+                        .filter((e) -> filter.apply(e.getKey(), e.getValue()))
+                        .map(Map.Entry::getKey)
+                        .collect(Collectors.toList());
+        }
+
+        Set<File> getTemporaryFiles(File file)
+        {
+            try (TransactionData txn = TransactionData.make(file))
+            {
+                maybeFail(txn.readLogFile(null));
+                return txn.getTemporaryFiles();
+            }
+            catch(Throwable t)
+            {
+                // We always fail if the onTxnErr is set to THROW or if we haven't
+                // reached the maximum number of attempts yet. Otherwise
+                // we just log an error and continue as if the txn log file does not exist
+                // clients can choose which behavior they want via onTxnLogError
+                if (attempts < MAX_ATTEMPTS ||
+                    onTxnErr == OnTxnErr.THROW)
+                    throw new RuntimeException(t);
+
+                logger.error("Failed to read temporary files of txn log {}", file, t);
+                return null; // txn.getTemporaryFiles() could be empty so we must use null to differentiate
+            }
+        }
+    }
+
+    @VisibleForTesting
+    static Set<File> getTemporaryFiles(CFMetaData metadata, File folder)
+    {
+        Set<File> ret = new HashSet<>();
+
+        List<File> directories = new Directories(metadata).getCFDirectories();
+        directories.add(folder);
+        for (File dir : directories)
+            ret.addAll(new FileLister(dir.toPath(),
+                                      (file, type) -> type != FileType.FINAL,
+                                      OnTxnErr.IGNORE).list());
+
+        return ret;
+    }
+}


[2/7] cassandra git commit: Improve transaction log under FS corruption

Posted by be...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/TransactionLogs.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/TransactionLogs.java b/src/java/org/apache/cassandra/db/lifecycle/TransactionLogs.java
deleted file mode 100644
index 821f58c..0000000
--- a/src/java/org/apache/cassandra/db/lifecycle/TransactionLogs.java
+++ /dev/null
@@ -1,786 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.lifecycle;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.util.*;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang3.StringUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.cassandra.utils.Throwables.merge;
-
-import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.CLibrary;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.UUIDGen;
-import org.apache.cassandra.utils.concurrent.Blocker;
-import org.apache.cassandra.utils.concurrent.Ref;
-import org.apache.cassandra.utils.concurrent.RefCounted;
-import org.apache.cassandra.utils.concurrent.Transactional;
-
-/**
- * IMPORTANT: When this object is involved in a transactional graph, and is not encapsulated in a LifecycleTransaction,
- * for correct behaviour its commit MUST occur before any others, since it may legitimately fail. This is consistent
- * with the Transactional API, which permits one failing action to occur at the beginning of the commit phase, but also
- * *requires* that the prepareToCommit() phase only take actions that can be rolled back.
- *
- * A class that tracks sstable files involved in a transaction across sstables:
- * if the transaction succeeds the old files should be deleted and the new ones kept; vice-versa if it fails.
- *
- * Two log files, NEW and OLD, contain new and old sstable files respectively. The log files also track each
- * other by referencing each others path in the contents.
- *
- * If the transaction finishes successfully:
- * - the OLD transaction file is deleted along with its contents, this includes the NEW transaction file.
- *   Before deleting we must let the SSTableTidier instances run first for any old readers that are being obsoleted
- *   (mark as compacted) by the transaction, see LifecycleTransaction
- *
- * If the transaction is aborted:
- * - the NEW transaction file and its contents are deleted, this includes the OLD transaction file
- *
- * On start-up:
- * - If we find a NEW transaction file, it means the transaction did not complete and we delete the NEW file and its contents
- * - If we find an OLD transaction file but not a NEW file, it means the transaction must have completed and so we delete
- *   all the contents of the OLD file, if they still exist, and the OLD file itself.
- *
- * See CASSANDRA-7066 for full details.
- */
-public class TransactionLogs extends Transactional.AbstractTransactional implements Transactional
-{
-    private static final Logger logger = LoggerFactory.getLogger(TransactionLogs.class);
-
-    /**
-     * A single transaction log file, either NEW or OLD.
-     */
-    final static class TransactionFile
-    {
-        static String EXT = ".log";
-        static char SEP = '_';
-        static String REGEX_STR = String.format("^(.*)_(.*)_(%s|%s)%s$", Type.NEW.txt, Type.OLD.txt, EXT);
-        static Pattern REGEX = Pattern.compile(REGEX_STR); //(opname)_(id)_(new|old).data
-
-        public enum Type
-        {
-            NEW (0, "new"),
-            OLD (1, "old");
-
-            public final int idx;
-            public final String txt;
-
-            Type(int idx, String txt)
-            {
-                this.idx = idx;
-                this.txt = txt;
-            }
-        };
-
-        public final Type type;
-        public final File file;
-        public final TransactionData parent;
-        public final Set<String> lines = new HashSet<>();
-
-        public TransactionFile(Type type, TransactionData parent)
-        {
-            this.type = type;
-            this.file = new File(parent.getFileName(type));
-            this.parent = parent;
-
-            if (exists())
-                lines.addAll(FileUtils.readLines(file));
-        }
-
-        public boolean add(SSTable table)
-        {
-            return add(table.descriptor.baseFilename());
-        }
-
-        private boolean add(String path)
-        {
-            String relativePath = FileUtils.getRelativePath(parent.getParentFolder(), path);
-            if (lines.contains(relativePath))
-                return false;
-
-            lines.add(relativePath);
-            FileUtils.append(file, relativePath);
-            return true;
-        }
-
-        public void remove(SSTable table)
-        {
-            String relativePath = FileUtils.getRelativePath(parent.getParentFolder(), table.descriptor.baseFilename());
-            assert lines.contains(relativePath) : String.format("%s is not tracked by %s", relativePath, file);
-
-            lines.remove(relativePath);
-            delete(relativePath);
-        }
-
-        public boolean contains(SSTable table)
-        {
-            String relativePath = FileUtils.getRelativePath(parent.getParentFolder(), table.descriptor.baseFilename());
-            return lines.contains(relativePath);
-        }
-
-        private void deleteContents()
-        {
-            deleteOpposite();
-
-            // we sync the parent file descriptor between opposite log deletion and
-            // contents deletion to ensure there is a happens before edge between them
-            parent.sync();
-
-            lines.forEach(line -> delete(line));
-            lines.clear();
-        }
-
-        private void deleteOpposite()
-        {
-            Type oppositeType = type == Type.NEW ? Type.OLD : Type.NEW;
-            String oppositeFile = FileUtils.getRelativePath(parent.getParentFolder(), parent.getFileName(oppositeType));
-            assert lines.contains(oppositeFile) : String.format("Could not find %s amongst lines", oppositeFile);
-
-            delete(oppositeFile);
-            lines.remove(oppositeFile);
-        }
-
-        private void delete(String relativePath)
-        {
-            getTrackedFiles(relativePath).forEach(file -> TransactionLogs.delete(file));
-        }
-
-        public Set<File> getTrackedFiles()
-        {
-            Set<File> ret = new HashSet<>();
-            FileUtils.readLines(file).forEach(line -> ret.addAll(getTrackedFiles(line)));
-            ret.add(file);
-            return ret;
-        }
-
-        private List<File> getTrackedFiles(String relativePath)
-        {
-            List<File> ret = new ArrayList<>();
-            File file = new File(StringUtils.join(parent.getParentFolder(), File.separator, relativePath));
-            if (file.exists())
-                ret.add(file);
-            else
-                ret.addAll(Arrays.asList(new File(parent.getParentFolder()).listFiles((dir, name) -> {
-                    return name.startsWith(relativePath);
-                })));
-
-            return ret;
-        }
-
-        public void delete(boolean deleteContents)
-        {
-            assert file.exists() : String.format("Expected %s to exists", file);
-
-            if (deleteContents)
-                deleteContents();
-
-            // we sync the parent file descriptor between contents and log deletion
-            // to ensure there is a happens before edge between them
-            parent.sync();
-
-            TransactionLogs.delete(file);
-        }
-
-        public boolean exists()
-        {
-            return file.exists();
-        }
-    }
-
-    /**
-     * We split the transaction data from the behavior because we need
-     * to reconstruct any left-overs and clean them up, as well as work
-     * out which files are temporary. So for these cases we don't want the full
-     * transactional behavior, plus it's handy for the TransactionTidier.
-     */
-    final static class TransactionData implements AutoCloseable
-    {
-        private final OperationType opType;
-        private final UUID id;
-        private final File folder;
-        private final TransactionFile[] files;
-        private int folderDescriptor;
-        private boolean succeeded;
-
-        static TransactionData make(File logFile)
-        {
-            Matcher matcher = TransactionFile.REGEX.matcher(logFile.getName());
-            assert matcher.matches();
-
-            OperationType operationType = OperationType.fromFileName(matcher.group(1));
-            UUID id = UUID.fromString(matcher.group(2));
-
-            return new TransactionData(operationType, logFile.getParentFile(), id);
-        }
-
-        TransactionData(OperationType opType, File folder, UUID id)
-        {
-            this.opType = opType;
-            this.id = id;
-            this.folder = folder;
-            this.files = new TransactionFile[TransactionFile.Type.values().length];
-            for (TransactionFile.Type t : TransactionFile.Type.values())
-                this.files[t.idx] = new TransactionFile(t, this);
-
-            this.folderDescriptor = CLibrary.tryOpenDirectory(folder.getPath());
-            this.succeeded = !newLog().exists() && oldLog().exists();
-        }
-
-        public void succeeded(boolean succeeded)
-        {
-            this.succeeded = succeeded;
-        }
-
-        public void close()
-        {
-            if (folderDescriptor > 0)
-            {
-                CLibrary.tryCloseFD(folderDescriptor);
-                folderDescriptor = -1;
-            }
-        }
-
-        void crossReference()
-        {
-            newLog().add(oldLog().file.getPath());
-            oldLog().add(newLog().file.getPath());
-        }
-
-        void sync()
-        {
-            if (folderDescriptor > 0)
-                CLibrary.trySync(folderDescriptor);
-        }
-
-        TransactionFile newLog()
-        {
-            return files[TransactionFile.Type.NEW.idx];
-        }
-
-        TransactionFile oldLog()
-        {
-            return files[TransactionFile.Type.OLD.idx];
-        }
-
-        OperationType getType()
-        {
-            return opType;
-        }
-
-        UUID getId()
-        {
-            return id;
-        }
-
-        Throwable removeUnfinishedLeftovers(Throwable accumulate)
-        {
-            try
-            {
-                if (succeeded)
-                    oldLog().delete(true);
-                else
-                    newLog().delete(true);
-            }
-            catch (Throwable t)
-            {
-                accumulate = merge(accumulate, t);
-            }
-
-            return accumulate;
-        }
-
-        Set<File> getTemporaryFiles()
-        {
-            sync();
-
-            if (newLog().exists())
-                return newLog().getTrackedFiles();
-            else
-                return oldLog().getTrackedFiles();
-        }
-
-        String getFileName(TransactionFile.Type type)
-        {
-            String fileName = StringUtils.join(opType.fileName,
-                                               TransactionFile.SEP,
-                                               id.toString(),
-                                               TransactionFile.SEP,
-                                               type.txt,
-                                               TransactionFile.EXT);
-            return StringUtils.join(folder, File.separator, fileName);
-        }
-
-        String getParentFolder()
-        {
-            return folder.getParent();
-        }
-
-        static boolean isLogFile(String name)
-        {
-            return TransactionFile.REGEX.matcher(name).matches();
-        }
-    }
-
-    private final Tracker tracker;
-    private final TransactionData data;
-    private final Ref<TransactionLogs> selfRef;
-    // Deleting sstables is tricky because the mmapping might not have been finalized yet,
-    // and delete will fail (on Windows) until it is (we only force the unmapping on SUN VMs).
-    // Additionally, we need to make sure to delete the data file first, so on restart the others
-    // will be recognized as GCable.
-    private static final Queue<Runnable> failedDeletions = new ConcurrentLinkedQueue<>();
-    private static final Blocker blocker = new Blocker();
-
-    TransactionLogs(OperationType opType, CFMetaData metadata)
-    {
-        this(opType, metadata, null);
-    }
-
-    TransactionLogs(OperationType opType, CFMetaData metadata, Tracker tracker)
-    {
-        this(opType, new Directories(metadata), tracker);
-    }
-
-    TransactionLogs(OperationType opType, Directories directories, Tracker tracker)
-    {
-        this(opType, directories.getDirectoryForNewSSTables(), tracker);
-    }
-
-    TransactionLogs(OperationType opType, File folder, Tracker tracker)
-    {
-        this.tracker = tracker;
-        this.data = new TransactionData(opType,
-                                        Directories.getTransactionsDirectory(folder),
-                                        UUIDGen.getTimeUUID());
-        this.selfRef = new Ref<>(this, new TransactionTidier(data));
-
-        data.crossReference();
-        if (logger.isDebugEnabled())
-            logger.debug("Created transaction logs with id {}", data.id);
-    }
-
-    /**
-     * Track a reader as new.
-     **/
-    void trackNew(SSTable table)
-    {
-        if (!data.newLog().add(table))
-            throw new IllegalStateException(table + " is already tracked as new");
-
-        data.newLog().add(table);
-    }
-
-    /**
-     * Stop tracking a reader as new.
-     */
-    void untrackNew(SSTable table)
-    {
-        data.newLog().remove(table);
-    }
-
-    /**
-     * Schedule a reader for deletion as soon as it is fully unreferenced and the transaction
-     * has been committed.
-     */
-    SSTableTidier obsoleted(SSTableReader reader)
-    {
-        if (data.newLog().contains(reader))
-        {
-            if (data.oldLog().contains(reader))
-                throw new IllegalArgumentException();
-
-            return new SSTableTidier(reader, true, this);
-        }
-
-        if (!data.oldLog().add(reader))
-            throw new IllegalStateException();
-
-        if (tracker != null)
-            tracker.notifyDeleting(reader);
-
-        return new SSTableTidier(reader, false, this);
-    }
-
-    OperationType getType()
-    {
-        return data.getType();
-    }
-
-    UUID getId()
-    {
-        return data.getId();
-    }
-
-    @VisibleForTesting
-    String getDataFolder()
-    {
-        return data.getParentFolder();
-    }
-
-    @VisibleForTesting
-    String getLogsFolder()
-    {
-        return StringUtils.join(getDataFolder(), File.separator, Directories.TRANSACTIONS_SUBDIR);
-    }
-
-    @VisibleForTesting
-    TransactionData getData()
-    {
-        return data;
-    }
-
-    private static void delete(File file)
-    {
-        try
-        {
-            if (logger.isDebugEnabled())
-                logger.debug("Deleting {}", file);
-
-            Files.delete(file.toPath());
-        }
-        catch (NoSuchFileException e)
-        {
-            logger.error("Unable to delete {} as it does not exist", file);
-        }
-        catch (IOException e)
-        {
-            logger.error("Unable to delete {}", file, e);
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * The transaction tidier.
-     *
-     * When the transaction reference is fully released we try to delete all the obsolete files
-     * depending on the transaction result.
-     */
-    private static class TransactionTidier implements RefCounted.Tidy, Runnable
-    {
-        private final TransactionData data;
-
-        public TransactionTidier(TransactionData data)
-        {
-            this.data = data;
-        }
-
-        public void tidy() throws Exception
-        {
-            run();
-        }
-
-        public String name()
-        {
-            return data.id.toString();
-        }
-
-        public void run()
-        {
-            if (logger.isDebugEnabled())
-                logger.debug("Removing files for transaction {}", name());
-
-            Throwable err = data.removeUnfinishedLeftovers(null);
-
-            if (err != null)
-            {
-                logger.info("Failed deleting files for transaction {}, we'll retry after GC and on on server restart", name(), err);
-                failedDeletions.add(this);
-            }
-            else
-            {
-                if (logger.isDebugEnabled())
-                    logger.debug("Closing file transaction {}", name());
-                data.close();
-            }
-        }
-    }
-
-    static class Obsoletion
-    {
-        final SSTableReader reader;
-        final SSTableTidier tidier;
-
-        public Obsoletion(SSTableReader reader, SSTableTidier tidier)
-        {
-            this.reader = reader;
-            this.tidier = tidier;
-        }
-    }
-
-    /**
-     * The SSTableReader tidier. When a reader is fully released and no longer referenced
-     * by any one, we run this. It keeps a reference to the parent transaction and releases
-     * it when done, so that the final transaction cleanup can run when all obsolete readers
-     * are released.
-     */
-    public static class SSTableTidier implements Runnable
-    {
-        // must not retain a reference to the SSTableReader, else leak detection cannot kick in
-        private final Descriptor desc;
-        private final long sizeOnDisk;
-        private final Tracker tracker;
-        private final boolean wasNew;
-        private final Ref<TransactionLogs> parentRef;
-
-        public SSTableTidier(SSTableReader referent, boolean wasNew, TransactionLogs parent)
-        {
-            this.desc = referent.descriptor;
-            this.sizeOnDisk = referent.bytesOnDisk();
-            this.tracker = parent.tracker;
-            this.wasNew = wasNew;
-            this.parentRef = parent.selfRef.tryRef();
-        }
-
-        public void run()
-        {
-            blocker.ask();
-
-            SystemKeyspace.clearSSTableReadMeter(desc.ksname, desc.cfname, desc.generation);
-
-            try
-            {
-                // If we can't successfully delete the DATA component, set the task to be retried later: see TransactionTidier
-                File datafile = new File(desc.filenameFor(Component.DATA));
-
-                delete(datafile);
-                // let the remainder be cleaned up by delete
-                SSTable.delete(desc, SSTable.discoverComponentsFor(desc));
-            }
-            catch (Throwable t)
-            {
-                logger.error("Failed deletion for {}, we'll retry after GC and on server restart", desc);
-                failedDeletions.add(this);
-                return;
-            }
-
-            if (tracker != null && tracker.cfstore != null && !wasNew)
-                tracker.cfstore.metric.totalDiskSpaceUsed.dec(sizeOnDisk);
-
-            // release the referent to the parent so that the all transaction files can be released
-            parentRef.release();
-        }
-
-        public void abort()
-        {
-            parentRef.release();
-        }
-    }
-
-    /**
-     * Retry all deletions that failed the first time around (presumably b/c the sstable was still mmap'd.)
-     * Useful because there are times when we know GC has been invoked; also exposed as an mbean.
-     */
-    public static void rescheduleFailedDeletions()
-    {
-        Runnable task;
-        while ( null != (task = failedDeletions.poll()))
-            ScheduledExecutors.nonPeriodicTasks.submit(task);
-    }
-
-    /**
-     * Deletions run on the nonPeriodicTasks executor, (both failedDeletions or global tidiers in SSTableReader)
-     * so by scheduling a new empty task and waiting for it we ensure any prior deletion has completed.
-     */
-    public static void waitForDeletions()
-    {
-        FBUtilities.waitOnFuture(ScheduledExecutors.nonPeriodicTasks.schedule(() -> {
-        }, 0, TimeUnit.MILLISECONDS));
-    }
-
-    @VisibleForTesting
-    public static void pauseDeletions(boolean stop)
-    {
-        blocker.block(stop);
-    }
-
-    private Throwable complete(Throwable accumulate)
-    {
-        try
-        {
-            try
-            {
-                if (data.succeeded)
-                    data.newLog().delete(false);
-                else
-                    data.oldLog().delete(false);
-            }
-            catch (Throwable t)
-            {
-                accumulate = merge(accumulate, t);
-            }
-
-            accumulate = selfRef.ensureReleased(accumulate);
-            return accumulate;
-        }
-        catch (Throwable t)
-        {
-            logger.error("Failed to complete file transaction {}", getId(), t);
-            return Throwables.merge(accumulate, t);
-        }
-    }
-
-    protected Throwable doCommit(Throwable accumulate)
-    {
-        data.succeeded(true);
-        return complete(accumulate);
-    }
-
-    protected Throwable doAbort(Throwable accumulate)
-    {
-        data.succeeded(false);
-        return complete(accumulate);
-    }
-
-    protected void doPrepare() { }
-
-    /**
-     * Called on startup to scan existing folders for any unfinished leftovers of
-     * operations that were ongoing when the process exited.
-     *
-     * We check if the new transaction file exists first, and if so we clean it up
-     * along with its contents, which includes the old file, else if only the old file exists
-     * it means the operation has completed and we only cleanup the old file with its contents.
-     */
-    static void removeUnfinishedLeftovers(CFMetaData metadata)
-    {
-        Throwable accumulate = null;
-        Set<UUID> ids = new HashSet<>();
-
-        for (File dir : getFolders(metadata, null))
-        {
-            File[] logs = dir.listFiles((dir1, name) -> {
-                return TransactionData.isLogFile(name);
-            });
-
-            for (File log : logs)
-            {
-                try (TransactionData data = TransactionData.make(log))
-                {
-                    // we need to check this because there are potentially 2 log files per operation
-                    if (ids.contains(data.id))
-                        continue;
-
-                    ids.add(data.id);
-                    accumulate = data.removeUnfinishedLeftovers(accumulate);
-                }
-            }
-        }
-
-        if (accumulate != null)
-            logger.error("Failed to remove unfinished transaction leftovers", accumulate);
-    }
-
-    /**
-     * Return a set of files that are temporary, that is they are involved with
-     * a transaction that hasn't completed yet.
-     *
-     * Only return the files that exist and that are located in the folder
-     * specified as a parameter or its sub-folders.
-     */
-    static Set<File> getTemporaryFiles(CFMetaData metadata, File folder)
-    {
-        Set<File> ret = new HashSet<>();
-        Set<UUID> ids = new HashSet<>();
-
-        for (File dir : getFolders(metadata, folder))
-        {
-            File[] logs = dir.listFiles((dir1, name) -> {
-                return TransactionData.isLogFile(name);
-            });
-
-            for (File log : logs)
-            {
-                try(TransactionData data = TransactionData.make(log))
-                {
-                    // we need to check this because there are potentially 2 log files per transaction
-                    if (ids.contains(data.id))
-                        continue;
-
-                    ids.add(data.id);
-                    ret.addAll(data.getTemporaryFiles()
-                                   .stream()
-                                   .filter(file -> FileUtils.isContained(folder, file))
-                                   .collect(Collectors.toSet()));
-                }
-            }
-        }
-
-        return ret;
-    }
-
-    /**
-     * Return the transaction log files that currently exist for this table.
-     */
-    static Set<File> getLogFiles(CFMetaData metadata)
-    {
-        Set<File> ret = new HashSet<>();
-        for (File dir : getFolders(metadata, null))
-            ret.addAll(Arrays.asList(dir.listFiles((dir1, name) -> {
-                return TransactionData.isLogFile(name);
-            })));
-
-        return ret;
-    }
-
-    /**
-     * A utility method to work out the existing transaction sub-folders
-     * either for a table, or a specific parent folder, or both.
-     */
-    private static List<File> getFolders(CFMetaData metadata, File folder)
-    {
-        List<File> ret = new ArrayList<>();
-        if (metadata != null)
-        {
-            Directories directories = new Directories(metadata);
-            ret.addAll(directories.getExistingDirectories(Directories.TRANSACTIONS_SUBDIR));
-        }
-
-        if (folder != null)
-        {
-            File opDir = Directories.getExistingDirectory(folder, Directories.TRANSACTIONS_SUBDIR);
-            if (opDir != null)
-                ret.add(opDir);
-        }
-
-        return ret;
-    }
-}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
index 20c3962..3286522 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.io.sstable;
 
 import java.io.File;
-import java.io.FilenameFilter;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.*;
@@ -27,16 +26,13 @@ import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 
 import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.streaming.*;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Pair;
 
@@ -77,89 +73,79 @@ public class SSTableLoader implements StreamEventHandler
     {
         outputHandler.output("Opening sstables and calculating sections to stream");
 
-        directory.list(new FilenameFilter()
-        {
-            final Map<File, Set<File>> allTemporaryFiles = new HashMap<>();
-            public boolean accept(File dir, String name)
-            {
-                File file = new File(dir, name);
-
-                if (file.isDirectory())
-                    return false;
-
-                Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
-                Descriptor desc = p == null ? null : p.left;
-                if (p == null || !p.right.equals(Component.DATA))
-                    return false;
-
-                if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists())
-                {
-                    outputHandler.output(String.format("Skipping file %s because index is missing", name));
-                    return false;
-                }
-
-                CFMetaData metadata = client.getTableMetadata(desc.cfname);
-                if (metadata == null)
-                {
-                    outputHandler.output(String.format("Skipping file %s: table %s.%s doesn't exist", name, keyspace, desc.cfname));
-                    return false;
-                }
-
-                Set<File> temporaryFiles = allTemporaryFiles.get(dir);
-                if (temporaryFiles == null)
-                {
-                    temporaryFiles = LifecycleTransaction.getTemporaryFiles(metadata, dir);
-                    allTemporaryFiles.put(dir, temporaryFiles);
-                }
-
-                if (temporaryFiles.contains(file))
-                {
-                    outputHandler.output(String.format("Skipping temporary file %s", name));
-                    return false;
-                }
-
-                Set<Component> components = new HashSet<>();
-                components.add(Component.DATA);
-                components.add(Component.PRIMARY_INDEX);
-                if (new File(desc.filenameFor(Component.SUMMARY)).exists())
-                    components.add(Component.SUMMARY);
-                if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
-                    components.add(Component.COMPRESSION_INFO);
-                if (new File(desc.filenameFor(Component.STATS)).exists())
-                    components.add(Component.STATS);
-
-                try
-                {
-                    // To conserve memory, open SSTableReaders without bloom filters and discard
-                    // the index summary after calculating the file sections to stream and the estimated
-                    // number of keys for each endpoint. See CASSANDRA-5555 for details.
-                    SSTableReader sstable = SSTableReader.openForBatch(desc, components, metadata);
-                    sstables.add(sstable);
-
-                    // calculate the sstable sections to stream as well as the estimated number of
-                    // keys per host
-                    for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : ranges.entrySet())
-                    {
-                        InetAddress endpoint = entry.getKey();
-                        Collection<Range<Token>> tokenRanges = entry.getValue();
-
-                        List<Pair<Long, Long>> sstableSections = sstable.getPositionsForRanges(tokenRanges);
-                        long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges);
-                        Ref<SSTableReader> ref = sstable.ref();
-                        StreamSession.SSTableStreamingSections details = new StreamSession.SSTableStreamingSections(ref, sstableSections, estimatedKeys, ActiveRepairService.UNREPAIRED_SSTABLE);
-                        streamingDetails.put(endpoint, details);
-                    }
-
-                    // to conserve heap space when bulk loading
-                    sstable.releaseSummary();
-                }
-                catch (IOException e)
-                {
-                    outputHandler.output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
-                }
-                return false;
-            }
-        });
+        LifecycleTransaction.getFiles(directory.toPath(),
+                                      (file, type) ->
+                                      {
+                                          File dir = file.getParentFile();
+                                          String name = file.getName();
+
+                                          if (type != Directories.FileType.FINAL)
+                                          {
+                                              outputHandler.output(String.format("Skipping temporary file %s", name));
+                                              return false;
+                                          }
+
+                                          Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
+                                          Descriptor desc = p == null ? null : p.left;
+                                          if (p == null || !p.right.equals(Component.DATA))
+                                              return false;
+
+                                          if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists())
+                                          {
+                                              outputHandler.output(String.format("Skipping file %s because index is missing", name));
+                                              return false;
+                                          }
+
+                                          CFMetaData metadata = client.getTableMetadata(desc.cfname);
+                                          if (metadata == null)
+                                          {
+                                              outputHandler.output(String.format("Skipping file %s: table %s.%s doesn't exist", name, keyspace, desc.cfname));
+                                              return false;
+                                          }
+
+                                          Set<Component> components = new HashSet<>();
+                                          components.add(Component.DATA);
+                                          components.add(Component.PRIMARY_INDEX);
+                                          if (new File(desc.filenameFor(Component.SUMMARY)).exists())
+                                              components.add(Component.SUMMARY);
+                                          if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
+                                              components.add(Component.COMPRESSION_INFO);
+                                          if (new File(desc.filenameFor(Component.STATS)).exists())
+                                              components.add(Component.STATS);
+
+                                          try
+                                          {
+                                              // To conserve memory, open SSTableReaders without bloom filters and discard
+                                              // the index summary after calculating the file sections to stream and the estimated
+                                              // number of keys for each endpoint. See CASSANDRA-5555 for details.
+                                              SSTableReader sstable = SSTableReader.openForBatch(desc, components, metadata);
+                                              sstables.add(sstable);
+
+                                              // calculate the sstable sections to stream as well as the estimated number of
+                                              // keys per host
+                                              for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : ranges.entrySet())
+                                              {
+                                                  InetAddress endpoint = entry.getKey();
+                                                  Collection<Range<Token>> tokenRanges = entry.getValue();
+
+                                                  List<Pair<Long, Long>> sstableSections = sstable.getPositionsForRanges(tokenRanges);
+                                                  long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges);
+                                                  Ref<SSTableReader> ref = sstable.ref();
+                                                  StreamSession.SSTableStreamingSections details = new StreamSession.SSTableStreamingSections(ref, sstableSections, estimatedKeys, ActiveRepairService.UNREPAIRED_SSTABLE);
+                                                  streamingDetails.put(endpoint, details);
+                                              }
+
+                                              // to conserve heap space when bulk loading
+                                              sstable.releaseSummary();
+                                          }
+                                          catch (IOException e)
+                                          {
+                                              outputHandler.output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
+                                          }
+                                          return false;
+                                      },
+                                      Directories.OnTxnErr.IGNORE);
+
         return sstables;
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index d5c192e..5502669 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -45,7 +45,7 @@ import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.index.SecondaryIndex;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.dht.*;
 import org.apache.cassandra.io.FSError;
@@ -1647,7 +1647,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
      * @return true if the this is the first time the file was marked obsolete.  Calling this
      * multiple times is usually buggy (see exceptions in Tracker.unmarkCompacting and removeOldSSTablesSize).
      */
-    public void markObsolete(TransactionLogs.SSTableTidier tidier)
+    public void markObsolete(TransactionLog.SSTableTidier tidier)
     {
         if (logger.isDebugEnabled())
             logger.debug("Marking {} compacted", getFilename());
@@ -1903,22 +1903,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
              : (sstableMetadata.totalRows == 0 ? 0 : (int)(sstableMetadata.totalColumnsSet / sstableMetadata.totalRows));
     }
 
-    public Set<Integer> getAncestors()
-    {
-        try
-        {
-            CompactionMetadata compactionMetadata = (CompactionMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.COMPACTION);
-            if (compactionMetadata != null)
-                return compactionMetadata.ancestors;
-            return Collections.emptySet();
-        }
-        catch (IOException e)
-        {
-            SSTableReader.logOpenException(descriptor, e);
-            return Collections.emptySet();
-        }
-    }
-
     public int getSSTableLevel()
     {
         return sstableMetadata.sstableLevel;
@@ -2191,7 +2175,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
         // sstable have been released
         private ScheduledFuture readMeterSyncFuture;
         // shared state managing if the logical sstable has been compacted; this is used in cleanup
-        private volatile TransactionLogs.SSTableTidier obsoletion;
+        private volatile TransactionLog.SSTableTidier obsoletion;
 
         GlobalTidy(final SSTableReader reader)
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index 9ef0b43..4fcf055 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -66,6 +66,8 @@ public abstract class Version
 
     public abstract boolean hasOldBfHashOrder();
 
+    public abstract boolean hasCompactionAncestors();
+
     public String getVersion()
     {
         return version;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 6df4b1e..d65710e 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -140,6 +140,11 @@ public class BigFormat implements SSTableFormat
          */
         private final boolean hasOldBfHashOrder;
 
+        /**
+         * CASSANDRA-7066: compaction ancerstors are no longer used and have been removed.
+         */
+        private final boolean hasCompactionAncestors;
+
         BigVersion(String version)
         {
             super(instance, version);
@@ -166,6 +171,7 @@ public class BigFormat implements SSTableFormat
             newFileName = version.compareTo("la") >= 0;
 
             hasOldBfHashOrder = version.compareTo("ma") < 0;
+            hasCompactionAncestors = version.compareTo("ma") < 0;
             storeRows = version.compareTo("ma") >= 0;
             correspondingMessagingVersion = storeRows
                                           ? MessagingService.VERSION_30
@@ -221,6 +227,12 @@ public class BigFormat implements SSTableFormat
         }
 
         @Override
+        public boolean hasCompactionAncestors()
+        {
+            return hasCompactionAncestors;
+        }
+
+        @Override
         public boolean hasNewFileName()
         {
             return newFileName;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index 29cbe5b..fa94ead 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@ -18,8 +18,6 @@
 package org.apache.cassandra.io.sstable.metadata;
 
 import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
 
 import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
 import com.clearspring.analytics.stream.cardinality.ICardinality;
@@ -39,13 +37,10 @@ public class CompactionMetadata extends MetadataComponent
 {
     public static final IMetadataComponentSerializer serializer = new CompactionMetadataSerializer();
 
-    public final Set<Integer> ancestors;
-
     public final ICardinality cardinalityEstimator;
 
-    public CompactionMetadata(Set<Integer> ancestors, ICardinality cardinalityEstimator)
+    public CompactionMetadata(ICardinality cardinalityEstimator)
     {
-        this.ancestors = ancestors;
         this.cardinalityEstimator = cardinalityEstimator;
     }
 
@@ -57,48 +52,46 @@ public class CompactionMetadata extends MetadataComponent
     @Override
     public boolean equals(Object o)
     {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
+        if (this == o)
+            return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
 
-        CompactionMetadata that = (CompactionMetadata) o;
-        return ancestors == null ? that.ancestors == null : ancestors.equals(that.ancestors);
+        // keeping equals and hashCode as all classes inheriting from MetadataComponent
+        // implement them but we have really nothing to compare
+        return true;
     }
 
     @Override
     public int hashCode()
     {
-        return ancestors != null ? ancestors.hashCode() : 0;
+        // see comment in equals
+        return 31;
     }
 
     public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
     {
         public int serializedSize(CompactionMetadata component) throws IOException
         {
-            int size = 0;
-            size += TypeSizes.sizeof(component.ancestors.size());
-            for (int g : component.ancestors)
-                size += TypeSizes.sizeof(g);
             byte[] serializedCardinality = component.cardinalityEstimator.getBytes();
-            size += TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length;
-            return size;
+            return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length;
         }
 
         public void serialize(CompactionMetadata component, DataOutputPlus out) throws IOException
         {
-            out.writeInt(component.ancestors.size());
-            for (int g : component.ancestors)
-                out.writeInt(g);
             ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
         }
 
         public CompactionMetadata deserialize(Version version, DataInputPlus in) throws IOException
         {
-            int nbAncestors = in.readInt();
-            Set<Integer> ancestors = new HashSet<>(nbAncestors);
-            for (int i = 0; i < nbAncestors; i++)
-                ancestors.add(in.readInt());
+            if (version.hasCompactionAncestors())
+            { // skip ancestors
+                int nbAncestors = in.readInt();
+                in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
+            }
             ICardinality cardinality = HyperLogLogPlus.Builder.build(ByteBufferUtil.readBytes(in, in.readInt()));
-            return new CompactionMetadata(ancestors, cardinality);
+            return new CompactionMetadata(cardinality);
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 433c31a..53ba922 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@ -23,6 +23,7 @@ import java.util.*;
 
 import com.google.common.collect.Maps;
 
+import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
@@ -60,9 +61,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
         out.writeDouble(validation.bloomFilterFPChance);
         out.writeDouble(stats.compressionRatio);
         out.writeUTF(validation.partitioner);
-        out.writeInt(compaction.ancestors.size());
-        for (Integer g : compaction.ancestors)
-            out.writeInt(g);
+        out.writeInt(0); // compaction ancestors
         StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out);
         out.writeInt(stats.sstableLevel);
         out.writeInt(stats.minClusteringValues.size());
@@ -99,10 +98,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
                 double bloomFilterFPChance = in.readDouble();
                 double compressionRatio = in.readDouble();
                 String partitioner = in.readUTF();
-                int nbAncestors = in.readInt();
-                Set<Integer> ancestors = new HashSet<>(nbAncestors);
-                for (int i = 0; i < nbAncestors; i++)
-                    ancestors.add(in.readInt());
+                int nbAncestors = in.readInt(); //skip compaction ancestors
+                in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
                 StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
                 int sstableLevel = 0;
                 if (in.available() > 0)
@@ -143,7 +140,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
                                                      -1));
                 if (types.contains(MetadataType.COMPACTION))
                     components.put(MetadataType.COMPACTION,
-                                   new CompactionMetadata(ancestors, null));
+                                   new CompactionMetadata(null));
             }
         }
         return components;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 9b06b53..1c93f58 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -94,7 +94,6 @@ public class MetadataCollector implements PartitionStatisticsCollector
     protected final MinMaxIntTracker localDeletionTimeTracker = new MinMaxIntTracker(Cell.NO_DELETION_TIME, Cell.NO_DELETION_TIME);
     protected final MinMaxIntTracker ttlTracker = new MinMaxIntTracker(Cell.NO_TTL, Cell.NO_TTL);
     protected double compressionRatio = NO_COMPRESSION_RATIO;
-    protected Set<Integer> ancestors = new HashSet<>();
     protected StreamingHistogram estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogram();
     protected int sstableLevel;
     protected ByteBuffer[] minClusteringValues;
@@ -120,29 +119,12 @@ public class MetadataCollector implements PartitionStatisticsCollector
         this.maxClusteringValues = new ByteBuffer[comparator.size()];
     }
 
-    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level, boolean skipAncestors)
+    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
     {
         this(comparator);
 
         replayPosition(ReplayPosition.getReplayPosition(sstables));
         sstableLevel(level);
-        // Get the max timestamp of the precompacted sstables
-        // and adds generation of live ancestors
-        if (!skipAncestors)
-        {
-            for (SSTableReader sstable : sstables)
-            {
-                addAncestor(sstable.descriptor.generation);
-                for (Integer i : sstable.getAncestors())
-                    if (new File(sstable.descriptor.withGeneration(i).filenameFor(Component.DATA)).exists())
-                        addAncestor(i);
-            }
-        }
-    }
-
-    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
-    {
-        this(sstables, comparator, level, false);
     }
 
     public MetadataCollector addKey(ByteBuffer key)
@@ -237,12 +219,6 @@ public class MetadataCollector implements PartitionStatisticsCollector
         return this;
     }
 
-    public MetadataCollector addAncestor(int generation)
-    {
-        this.ancestors.add(generation);
-        return this;
-    }
-
     public MetadataCollector sstableLevel(int sstableLevel)
     {
         this.sstableLevel = sstableLevel;
@@ -313,7 +289,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
                                                              repairedAt,
                                                              totalColumnsSet,
                                                              totalRows));
-        components.put(MetadataType.COMPACTION, new CompactionMetadata(ancestors, cardinality));
+        components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality));
         components.put(MetadataType.HEADER, header.toComponent());
         return components;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index c3de1db..0ba2307 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -616,11 +616,6 @@ public class FileUtils
         {
             return Files.readAllLines(file.toPath(), Charset.forName("utf-8"));
         }
-        catch (NoSuchFileException ex)
-        {
-            logger.warn("Tried to read non existing file: {}", file);
-            return Collections.emptyList();
-        }
         catch (IOException ex)
         {
             throw new RuntimeException(ex);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/service/GCInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/GCInspector.java b/src/java/org/apache/cassandra/service/GCInspector.java
index fc7ff3d..252d1c3 100644
--- a/src/java/org/apache/cassandra/service/GCInspector.java
+++ b/src/java/org/apache/cassandra/service/GCInspector.java
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
 import com.sun.management.GarbageCollectionNotificationInfo;
 import com.sun.management.GcInfo;
 
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.utils.StatusLogger;
 
 public class GCInspector implements NotificationListener, GCInspectorMXBean
@@ -284,7 +284,7 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean
 
             // if we just finished an old gen collection and we're still using a lot of memory, try to reduce the pressure
             if (gcState.assumeGCIsOldGen)
-                TransactionLogs.rescheduleFailedDeletions();
+                TransactionLog.rescheduleFailedDeletions();
         }
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/service/StartupChecks.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java
index fee9789..9ffef96 100644
--- a/src/java/org/apache/cassandra/service/StartupChecks.java
+++ b/src/java/org/apache/cassandra/service/StartupChecks.java
@@ -237,8 +237,7 @@ public class StartupChecks
                 {
                     String name = dir.getFileName().toString();
                     return (name.equals(Directories.SNAPSHOT_SUBDIR)
-                            || name.equals(Directories.BACKUPS_SUBDIR)
-                            || name.equals(Directories.TRANSACTIONS_SUBDIR))
+                            || name.equals(Directories.BACKUPS_SUBDIR))
                            ? FileVisitResult.SKIP_SUBTREE
                            : FileVisitResult.CONTINUE;
                 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 959f7e3..5966e49 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -70,7 +70,7 @@ import org.apache.cassandra.config.*;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.dht.BootStrapper;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
@@ -4241,7 +4241,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     public void rescheduleFailedDeletions()
     {
-        TransactionLogs.rescheduleFailedDeletions();
+        TransactionLog.rescheduleFailedDeletions();
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
index bc17750..c8587d8 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
@@ -34,6 +34,7 @@ import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -68,7 +69,7 @@ public class SSTableExpiredBlockers
 
         Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
-        Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+        Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
         Set<SSTableReader> sstables = new HashSet<>();
         for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
index d73a325..cb3cc5c 100644
--- a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
@@ -23,6 +23,7 @@ import java.util.Set;
 
 import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
@@ -75,7 +76,7 @@ public class SSTableLevelResetter
             Keyspace keyspace = Keyspace.openWithoutSSTables(keyspaceName);
             ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnfamily);
             boolean foundSSTable = false;
-            for (Map.Entry<Descriptor, Set<Component>> sstable : cfs.directories.sstableLister().list().entrySet())
+            for (Map.Entry<Descriptor, Set<Component>> sstable : cfs.directories.sstableLister(Directories.OnTxnErr.THROW).list().entrySet())
             {
                 if (sstable.getValue().contains(Component.STATS))
                 {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index a644f49..7764bbf 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -78,9 +78,7 @@ public class SSTableMetadataViewer
                 }
                 if (compaction != null)
                 {
-                    out.printf("Ancestors: %s%n", compaction.ancestors.toString());
                     out.printf("Estimated cardinality: %s%n", compaction.cardinalityEstimator.cardinality());
-
                 }
             }
             else

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
index f7e477f..95f516a 100644
--- a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
+++ b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
@@ -95,7 +95,7 @@ public class SSTableOfflineRelevel
 
         Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
-        Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+        Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
         Set<SSTableReader> sstables = new HashSet<>();
         for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneLister.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneLister.java b/src/java/org/apache/cassandra/tools/StandaloneLister.java
deleted file mode 100644
index 71575c7..0000000
--- a/src/java/org/apache/cassandra/tools/StandaloneLister.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.tools;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.utils.OutputHandler;
-import org.apache.commons.cli.*;
-
-import java.io.File;
-
-import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
-
-public class StandaloneLister
-{
-    private static final String TOOL_NAME = "sstablelister";
-    private static final String TYPE_OPTION  = "type";
-    private static final String OP_LOG_OPTION  = "oplog";
-    private static final String VERBOSE_OPTION  = "verbose";
-    private static final String DEBUG_OPTION  = "debug";
-    private static final String HELP_OPTION  = "help";
-
-    public static void main(String args[])
-    {
-        Options options = Options.parseArgs(args);
-        try
-        {
-            // load keyspace descriptions.
-            Schema.instance.loadFromDisk(false);
-
-            CFMetaData metadata = Schema.instance.getCFMetaData(options.keyspaceName, options.cfName);
-            if (metadata == null)
-                throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
-                                                                 options.keyspaceName,
-                                                                 options.cfName));
-
-            OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
-
-            Directories directories = new Directories(metadata);
-            Directories.SSTableLister lister = directories.sstableLister();
-
-            if (options.type == Options.FileType.FINAL)
-                lister.skipTemporary(true);
-            else if (options.type == Options.FileType.TMP)
-                lister.onlyTemporary(true);
-
-            for (File file : lister.listFiles())
-               handler.output(file.getCanonicalPath());
-
-            if (options.oplogs)
-            {
-                for (File file : LifecycleTransaction.getLogFiles(metadata))
-                {
-                    handler.output(file.getCanonicalPath());
-                }
-            }
-
-            System.exit(0);
-        }
-        catch (Exception e)
-        {
-            System.err.println(e.getMessage());
-            if (options.debug)
-                e.printStackTrace(System.err);
-            System.exit(1);
-        }
-    }
-
-    private static class Options
-    {
-        public enum FileType
-        {
-            ALL("all", "list all files, final or temporary"),
-            TMP("tmp", "list temporary files only"),
-            FINAL("final", "list final files only");
-
-            public String option;
-            public String descr;
-            FileType(String option, String descr)
-            {
-                this.option = option;
-                this.descr = descr;
-            }
-
-            static FileType fromOption(String option)
-            {
-                for (FileType fileType : FileType.values())
-                {
-                    if (fileType.option.equals(option))
-                        return fileType;
-                }
-
-                return FileType.ALL;
-            }
-
-            static String descr()
-            {
-                StringBuilder str = new StringBuilder();
-                for (FileType fileType : FileType.values())
-                {
-                    str.append(fileType.option);
-                    str.append(" (");
-                    str.append(fileType.descr);
-                    str.append("), ");
-                }
-                return str.toString();
-            }
-        }
-
-        public final String keyspaceName;
-        public final String cfName;
-
-        public boolean debug;
-        public boolean verbose;
-        public boolean oplogs;
-        public FileType type;
-
-        private Options(String keyspaceName, String cfName)
-        {
-            this.keyspaceName = keyspaceName;
-            this.cfName = cfName;
-        }
-
-        public static Options parseArgs(String cmdArgs[])
-        {
-            CommandLineParser parser = new GnuParser();
-            CmdLineOptions options = getCmdLineOptions();
-            try
-            {
-                CommandLine cmd = parser.parse(options, cmdArgs, false);
-
-                if (cmd.hasOption(HELP_OPTION))
-                {
-                    printUsage(options);
-                    System.exit(0);
-                }
-
-                String[] args = cmd.getArgs();
-                if (args.length != 2)
-                {
-                    String msg = args.length < 2 ? "Missing arguments" : "Too many arguments";
-                    System.err.println(msg);
-                    printUsage(options);
-                    System.exit(1);
-                }
-
-                String keyspaceName = args[0];
-                String cfName = args[1];
-
-                Options opts = new Options(keyspaceName, cfName);
-
-                opts.debug = cmd.hasOption(DEBUG_OPTION);
-                opts.verbose = cmd.hasOption(VERBOSE_OPTION);
-                opts.type = FileType.fromOption(cmd.getOptionValue(TYPE_OPTION));
-                opts.oplogs = cmd.hasOption(OP_LOG_OPTION);
-
-                return opts;
-            }
-            catch (ParseException e)
-            {
-                errorMsg(e.getMessage(), options);
-                return null;
-            }
-        }
-
-        private static void errorMsg(String msg, CmdLineOptions options)
-        {
-            System.err.println(msg);
-            printUsage(options);
-            System.exit(1);
-        }
-
-        private static CmdLineOptions getCmdLineOptions()
-        {
-            CmdLineOptions options = new CmdLineOptions();
-            options.addOption("d", DEBUG_OPTION, "display stack traces");
-            options.addOption("h", HELP_OPTION, "display this help message");
-            options.addOption("o", OP_LOG_OPTION, "include operation logs");
-            options.addOption("t", TYPE_OPTION, true, FileType.descr());
-            options.addOption("v", VERBOSE_OPTION, "verbose output");
-
-            return options;
-        }
-
-        public static void printUsage(CmdLineOptions options)
-        {
-            String usage = String.format("%s [options] <keyspace> <column_family>", TOOL_NAME);
-            StringBuilder header = new StringBuilder();
-            header.append("--\n");
-            header.append("List sstable files for the provided table." );
-            header.append("\n--\n");
-            header.append("Options are:");
-            new HelpFormatter().printHelp(usage, header.toString(), options, "");
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java b/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
new file mode 100644
index 0000000..6e2be1d
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.tools;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.commons.cli.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.function.BiFunction;
+
+import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
+
+public class StandaloneSSTableUtil
+{
+    private static final String TOOL_NAME = "sstableutil";
+    private static final String TYPE_OPTION  = "type";
+    private static final String OP_LOG_OPTION  = "oplog";
+    private static final String VERBOSE_OPTION  = "verbose";
+    private static final String DEBUG_OPTION  = "debug";
+    private static final String HELP_OPTION  = "help";
+    private static final String CLEANUP_OPTION = "cleanup";
+
+    public static void main(String args[])
+    {
+        Options options = Options.parseArgs(args);
+        try
+        {
+            // load keyspace descriptions.
+            Schema.instance.loadFromDisk(false);
+
+            CFMetaData metadata = Schema.instance.getCFMetaData(options.keyspaceName, options.cfName);
+            if (metadata == null)
+                throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
+                                                                 options.keyspaceName,
+                                                                 options.cfName));
+
+            OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
+
+            if (options.cleanup)
+            {
+                handler.output("Cleanuping up...");
+                LifecycleTransaction.removeUnfinishedLeftovers(metadata);
+            }
+            else
+            {
+                handler.output("Listing files...");
+                listFiles(options, metadata, handler);
+            }
+
+            System.exit(0);
+        }
+        catch (Exception e)
+        {
+            System.err.println(e.getMessage());
+            if (options.debug)
+                e.printStackTrace(System.err);
+            System.exit(1);
+        }
+    }
+
+    private static void listFiles(Options options, CFMetaData metadata, OutputHandler handler) throws IOException
+    {
+        Directories directories = new Directories(metadata);
+
+        for (File dir : directories.getCFDirectories())
+        {
+            for (File file : LifecycleTransaction.getFiles(dir.toPath(), getFilter(options), Directories.OnTxnErr.THROW))
+                handler.output(file.getCanonicalPath());
+        }
+    }
+
+    private static BiFunction<File, Directories.FileType, Boolean> getFilter(Options options)
+    {
+        return (file, type) ->
+        {
+            switch(type)
+            {
+                case FINAL:
+                    return options.type != Options.FileType.TMP;
+                case TEMPORARY:
+                    return options.type != Options.FileType.FINAL;
+                case TXN_LOG:
+                    return options.oplogs;
+                default:
+                    throw new AssertionError();
+            }
+        };
+    }
+
+    private static class Options
+    {
+        public enum FileType
+        {
+            ALL("all", "list all files, final or temporary"),
+            TMP("tmp", "list temporary files only"),
+            FINAL("final", "list final files only");
+
+            public String option;
+            public String descr;
+            FileType(String option, String descr)
+            {
+                this.option = option;
+                this.descr = descr;
+            }
+
+            static FileType fromOption(String option)
+            {
+                for (FileType fileType : FileType.values())
+                {
+                    if (fileType.option.equals(option))
+                        return fileType;
+                }
+
+                return FileType.ALL;
+            }
+
+            static String descr()
+            {
+                StringBuilder str = new StringBuilder();
+                for (FileType fileType : FileType.values())
+                {
+                    str.append(fileType.option);
+                    str.append(" (");
+                    str.append(fileType.descr);
+                    str.append("), ");
+                }
+                return str.toString();
+            }
+        }
+
+        public final String keyspaceName;
+        public final String cfName;
+
+        public boolean debug;
+        public boolean verbose;
+        public boolean oplogs;
+        public boolean cleanup;
+        public FileType type;
+
+        private Options(String keyspaceName, String cfName)
+        {
+            this.keyspaceName = keyspaceName;
+            this.cfName = cfName;
+        }
+
+        public static Options parseArgs(String cmdArgs[])
+        {
+            CommandLineParser parser = new GnuParser();
+            CmdLineOptions options = getCmdLineOptions();
+            try
+            {
+                CommandLine cmd = parser.parse(options, cmdArgs, false);
+
+                if (cmd.hasOption(HELP_OPTION))
+                {
+                    printUsage(options);
+                    System.exit(0);
+                }
+
+                String[] args = cmd.getArgs();
+                if (args.length != 2)
+                {
+                    String msg = args.length < 2 ? "Missing arguments" : "Too many arguments";
+                    System.err.println(msg);
+                    printUsage(options);
+                    System.exit(1);
+                }
+
+                String keyspaceName = args[0];
+                String cfName = args[1];
+
+                Options opts = new Options(keyspaceName, cfName);
+
+                opts.debug = cmd.hasOption(DEBUG_OPTION);
+                opts.verbose = cmd.hasOption(VERBOSE_OPTION);
+                opts.type = FileType.fromOption(cmd.getOptionValue(TYPE_OPTION));
+                opts.oplogs = cmd.hasOption(OP_LOG_OPTION);
+                opts.cleanup = cmd.hasOption(CLEANUP_OPTION);
+
+                return opts;
+            }
+            catch (ParseException e)
+            {
+                errorMsg(e.getMessage(), options);
+                return null;
+            }
+        }
+
+        private static void errorMsg(String msg, CmdLineOptions options)
+        {
+            System.err.println(msg);
+            printUsage(options);
+            System.exit(1);
+        }
+
+        private static CmdLineOptions getCmdLineOptions()
+        {
+            CmdLineOptions options = new CmdLineOptions();
+            options.addOption("c", CLEANUP_OPTION, "clean-up any outstanding transactions");
+            options.addOption("d", DEBUG_OPTION, "display stack traces");
+            options.addOption("h", HELP_OPTION, "display this help message");
+            options.addOption("o", OP_LOG_OPTION, "include operation logs");
+            options.addOption("t", TYPE_OPTION, true, FileType.descr());
+            options.addOption("v", VERBOSE_OPTION, "verbose output");
+
+            return options;
+        }
+
+        public static void printUsage(CmdLineOptions options)
+        {
+            String usage = String.format("%s [options] <keyspace> <column_family>", TOOL_NAME);
+            StringBuilder header = new StringBuilder();
+            header.append("--\n");
+            header.append("List sstable files for the provided table." );
+            header.append("\n--\n");
+            header.append("Options are:");
+            new HelpFormatter().printHelp(usage, header.toString(), options, "");
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 9388d98..f64b8d9 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -34,7 +34,7 @@ import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.*;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.*;
 import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -84,7 +84,7 @@ public class StandaloneScrubber
             String snapshotName = "pre-scrub-" + System.currentTimeMillis();
 
             OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
-            Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+            Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
 
             List<SSTableReader> sstables = new ArrayList<>();
 
@@ -145,7 +145,7 @@ public class StandaloneScrubber
             // Check (and repair) manifests
             checkManifest(cfs.getCompactionStrategyManager(), cfs, sstables);
             CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             System.exit(0); // We need that to stop non daemonized threads
         }
         catch (Exception e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index e53038d..a192491 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@ -23,7 +23,7 @@ import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.commons.cli.*;
 
@@ -164,7 +164,7 @@ public class StandaloneSplitter
                 }
             }
             CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             System.exit(0); // We need that to stop non daemonized threads
         }
         catch (Exception e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 0851d5b..88e34b7 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -20,7 +20,7 @@ package org.apache.cassandra.tools;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.commons.cli.*;
 
@@ -63,7 +63,7 @@ public class StandaloneUpgrader
             ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cf);
 
             OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
-            Directories.SSTableLister lister = cfs.directories.sstableLister();
+            Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW);
             if (options.snapshot != null)
                 lister.onlyBackups(true).snapshots(options.snapshot);
             else
@@ -120,7 +120,7 @@ public class StandaloneUpgrader
                 }
             }
             CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             System.exit(0);
         }
         catch (Exception e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
index f71f58d..0b17e39 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
@@ -69,7 +69,7 @@ public class StandaloneVerifier
             ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cfName);
 
             OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
-            Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+            Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
 
             boolean extended = options.extended;
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/nodetool/Stop.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Stop.java b/src/java/org/apache/cassandra/tools/nodetool/Stop.java
index ad1fc27..6229e65 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Stop.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Stop.java
@@ -36,7 +36,7 @@ public class Stop extends NodeToolCmd
 
     @Option(title = "compactionId",
            name = {"-id", "--compaction-id"},
-           description = "Use -id to stop a compaction by the specified id. Ids can be found in the system.compactions_in_progress table.",
+           description = "Use -id to stop a compaction by the specified id. Ids can be found in the transaction log files whose name starts with compaction_, located in the table transactions folder.",
            required = false)
     private String compactionId = "";
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 47c07ac..9e36bab 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -532,7 +532,7 @@ public class ColumnFamilyStoreTest
 
         ColumnFamilyStore.scrubDataDirectories(cfs.metadata);
 
-        List<File> ssTableFiles = new Directories(cfs.metadata).sstableLister().listFiles();
+        List<File> ssTableFiles = new Directories(cfs.metadata).sstableLister(Directories.OnTxnErr.THROW).listFiles();
         assertNotNull(ssTableFiles);
         assertEquals(0, ssTableFiles.size());
     }


[7/7] cassandra git commit: Merge branch 'cassandra-3.0' into trunk

Posted by be...@apache.org.
Merge branch 'cassandra-3.0' into trunk


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/13ec86ce
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/13ec86ce
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/13ec86ce

Branch: refs/heads/trunk
Commit: 13ec86ce9e4fe6616bff85aa4730686212a347c1
Parents: 5e9835b 5726625
Author: Benedict Elliott Smith <be...@apache.org>
Authored: Mon Aug 17 10:20:15 2015 +0100
Committer: Benedict Elliott Smith <be...@apache.org>
Committed: Mon Aug 17 10:20:15 2015 +0100

----------------------------------------------------------------------
 NEWS.txt                                        |   18 +
 bin/sstablelister                               |   55 -
 bin/sstablelister.bat                           |   41 -
 bin/sstableutil                                 |   55 +
 bin/sstableutil.bat                             |   41 +
 .../apache/cassandra/db/ColumnFamilyStore.java  |   12 +-
 .../org/apache/cassandra/db/Directories.java    |  145 ++-
 .../db/compaction/CompactionManagerMBean.java   |    3 +-
 .../cassandra/db/compaction/Upgrader.java       |    9 -
 .../writers/MajorLeveledCompactionWriter.java   |    9 +-
 .../apache/cassandra/db/lifecycle/Helpers.java  |   12 +-
 .../db/lifecycle/LifecycleTransaction.java      |   69 +-
 .../apache/cassandra/db/lifecycle/Tracker.java  |    4 +-
 .../cassandra/db/lifecycle/TransactionLog.java  | 1137 ++++++++++++++++++
 .../cassandra/db/lifecycle/TransactionLogs.java |  786 ------------
 .../cassandra/io/sstable/SSTableLoader.java     |  162 ++-
 .../io/sstable/format/SSTableReader.java        |   22 +-
 .../cassandra/io/sstable/format/Version.java    |    2 +
 .../io/sstable/format/big/BigFormat.java        |   12 +
 .../io/sstable/metadata/CompactionMetadata.java |   43 +-
 .../metadata/LegacyMetadataSerializer.java      |   13 +-
 .../io/sstable/metadata/MetadataCollector.java  |   28 +-
 .../org/apache/cassandra/io/util/FileUtils.java |    5 -
 .../apache/cassandra/service/GCInspector.java   |    4 +-
 .../apache/cassandra/service/StartupChecks.java |    3 +-
 .../cassandra/service/StorageService.java       |    4 +-
 .../cassandra/tools/SSTableExpiredBlockers.java |    3 +-
 .../cassandra/tools/SSTableLevelResetter.java   |    3 +-
 .../cassandra/tools/SSTableMetadataViewer.java  |    2 -
 .../cassandra/tools/SSTableOfflineRelevel.java  |    2 +-
 .../cassandra/tools/StandaloneLister.java       |  214 ----
 .../cassandra/tools/StandaloneSSTableUtil.java  |  241 ++++
 .../cassandra/tools/StandaloneScrubber.java     |    6 +-
 .../cassandra/tools/StandaloneSplitter.java     |    4 +-
 .../cassandra/tools/StandaloneUpgrader.java     |    6 +-
 .../cassandra/tools/StandaloneVerifier.java     |    2 +-
 .../apache/cassandra/tools/nodetool/Stop.java   |    2 +-
 .../cassandra/db/ColumnFamilyStoreTest.java     |    2 +-
 .../apache/cassandra/db/DirectoriesTest.java    |    7 +-
 .../org/apache/cassandra/db/KeyCacheTest.java   |    6 +-
 .../unit/org/apache/cassandra/db/ScrubTest.java |    4 +-
 .../cassandra/db/lifecycle/HelpersTest.java     |    9 +-
 .../db/lifecycle/LifecycleTransactionTest.java  |    2 +-
 .../db/lifecycle/RealTransactionsTest.java      |   15 +-
 .../cassandra/db/lifecycle/TrackerTest.java     |   90 +-
 .../db/lifecycle/TransactionLogTest.java        |  791 ++++++++++++
 .../db/lifecycle/TransactionLogsTest.java       |  581 ---------
 .../io/sstable/CQLSSTableWriterClientTest.java  |    7 -
 .../io/sstable/CQLSSTableWriterTest.java        |    1 -
 .../io/sstable/SSTableRewriterTest.java         |   36 +-
 .../metadata/MetadataSerializerTest.java        |    7 -
 .../org/apache/cassandra/schema/DefsTest.java   |   11 +-
 52 files changed, 2625 insertions(+), 2123 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/13ec86ce/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------


[5/7] cassandra git commit: Improve transaction log under FS corruption

Posted by be...@apache.org.
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/db/lifecycle/TransactionLogs.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/lifecycle/TransactionLogs.java b/src/java/org/apache/cassandra/db/lifecycle/TransactionLogs.java
deleted file mode 100644
index 821f58c..0000000
--- a/src/java/org/apache/cassandra/db/lifecycle/TransactionLogs.java
+++ /dev/null
@@ -1,786 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.cassandra.db.lifecycle;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.util.*;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang3.StringUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.cassandra.utils.Throwables.merge;
-
-import org.apache.cassandra.concurrent.ScheduledExecutors;
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.SystemKeyspace;
-import org.apache.cassandra.db.compaction.OperationType;
-import org.apache.cassandra.io.sstable.Component;
-import org.apache.cassandra.io.sstable.Descriptor;
-import org.apache.cassandra.io.sstable.SSTable;
-import org.apache.cassandra.io.sstable.format.SSTableReader;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.cassandra.utils.CLibrary;
-import org.apache.cassandra.utils.FBUtilities;
-import org.apache.cassandra.utils.Throwables;
-import org.apache.cassandra.utils.UUIDGen;
-import org.apache.cassandra.utils.concurrent.Blocker;
-import org.apache.cassandra.utils.concurrent.Ref;
-import org.apache.cassandra.utils.concurrent.RefCounted;
-import org.apache.cassandra.utils.concurrent.Transactional;
-
-/**
- * IMPORTANT: When this object is involved in a transactional graph, and is not encapsulated in a LifecycleTransaction,
- * for correct behaviour its commit MUST occur before any others, since it may legitimately fail. This is consistent
- * with the Transactional API, which permits one failing action to occur at the beginning of the commit phase, but also
- * *requires* that the prepareToCommit() phase only take actions that can be rolled back.
- *
- * A class that tracks sstable files involved in a transaction across sstables:
- * if the transaction succeeds the old files should be deleted and the new ones kept; vice-versa if it fails.
- *
- * Two log files, NEW and OLD, contain new and old sstable files respectively. The log files also track each
- * other by referencing each others path in the contents.
- *
- * If the transaction finishes successfully:
- * - the OLD transaction file is deleted along with its contents, this includes the NEW transaction file.
- *   Before deleting we must let the SSTableTidier instances run first for any old readers that are being obsoleted
- *   (mark as compacted) by the transaction, see LifecycleTransaction
- *
- * If the transaction is aborted:
- * - the NEW transaction file and its contents are deleted, this includes the OLD transaction file
- *
- * On start-up:
- * - If we find a NEW transaction file, it means the transaction did not complete and we delete the NEW file and its contents
- * - If we find an OLD transaction file but not a NEW file, it means the transaction must have completed and so we delete
- *   all the contents of the OLD file, if they still exist, and the OLD file itself.
- *
- * See CASSANDRA-7066 for full details.
- */
-public class TransactionLogs extends Transactional.AbstractTransactional implements Transactional
-{
-    private static final Logger logger = LoggerFactory.getLogger(TransactionLogs.class);
-
-    /**
-     * A single transaction log file, either NEW or OLD.
-     */
-    final static class TransactionFile
-    {
-        static String EXT = ".log";
-        static char SEP = '_';
-        static String REGEX_STR = String.format("^(.*)_(.*)_(%s|%s)%s$", Type.NEW.txt, Type.OLD.txt, EXT);
-        static Pattern REGEX = Pattern.compile(REGEX_STR); //(opname)_(id)_(new|old).data
-
-        public enum Type
-        {
-            NEW (0, "new"),
-            OLD (1, "old");
-
-            public final int idx;
-            public final String txt;
-
-            Type(int idx, String txt)
-            {
-                this.idx = idx;
-                this.txt = txt;
-            }
-        };
-
-        public final Type type;
-        public final File file;
-        public final TransactionData parent;
-        public final Set<String> lines = new HashSet<>();
-
-        public TransactionFile(Type type, TransactionData parent)
-        {
-            this.type = type;
-            this.file = new File(parent.getFileName(type));
-            this.parent = parent;
-
-            if (exists())
-                lines.addAll(FileUtils.readLines(file));
-        }
-
-        public boolean add(SSTable table)
-        {
-            return add(table.descriptor.baseFilename());
-        }
-
-        private boolean add(String path)
-        {
-            String relativePath = FileUtils.getRelativePath(parent.getParentFolder(), path);
-            if (lines.contains(relativePath))
-                return false;
-
-            lines.add(relativePath);
-            FileUtils.append(file, relativePath);
-            return true;
-        }
-
-        public void remove(SSTable table)
-        {
-            String relativePath = FileUtils.getRelativePath(parent.getParentFolder(), table.descriptor.baseFilename());
-            assert lines.contains(relativePath) : String.format("%s is not tracked by %s", relativePath, file);
-
-            lines.remove(relativePath);
-            delete(relativePath);
-        }
-
-        public boolean contains(SSTable table)
-        {
-            String relativePath = FileUtils.getRelativePath(parent.getParentFolder(), table.descriptor.baseFilename());
-            return lines.contains(relativePath);
-        }
-
-        private void deleteContents()
-        {
-            deleteOpposite();
-
-            // we sync the parent file descriptor between opposite log deletion and
-            // contents deletion to ensure there is a happens before edge between them
-            parent.sync();
-
-            lines.forEach(line -> delete(line));
-            lines.clear();
-        }
-
-        private void deleteOpposite()
-        {
-            Type oppositeType = type == Type.NEW ? Type.OLD : Type.NEW;
-            String oppositeFile = FileUtils.getRelativePath(parent.getParentFolder(), parent.getFileName(oppositeType));
-            assert lines.contains(oppositeFile) : String.format("Could not find %s amongst lines", oppositeFile);
-
-            delete(oppositeFile);
-            lines.remove(oppositeFile);
-        }
-
-        private void delete(String relativePath)
-        {
-            getTrackedFiles(relativePath).forEach(file -> TransactionLogs.delete(file));
-        }
-
-        public Set<File> getTrackedFiles()
-        {
-            Set<File> ret = new HashSet<>();
-            FileUtils.readLines(file).forEach(line -> ret.addAll(getTrackedFiles(line)));
-            ret.add(file);
-            return ret;
-        }
-
-        private List<File> getTrackedFiles(String relativePath)
-        {
-            List<File> ret = new ArrayList<>();
-            File file = new File(StringUtils.join(parent.getParentFolder(), File.separator, relativePath));
-            if (file.exists())
-                ret.add(file);
-            else
-                ret.addAll(Arrays.asList(new File(parent.getParentFolder()).listFiles((dir, name) -> {
-                    return name.startsWith(relativePath);
-                })));
-
-            return ret;
-        }
-
-        public void delete(boolean deleteContents)
-        {
-            assert file.exists() : String.format("Expected %s to exists", file);
-
-            if (deleteContents)
-                deleteContents();
-
-            // we sync the parent file descriptor between contents and log deletion
-            // to ensure there is a happens before edge between them
-            parent.sync();
-
-            TransactionLogs.delete(file);
-        }
-
-        public boolean exists()
-        {
-            return file.exists();
-        }
-    }
-
-    /**
-     * We split the transaction data from the behavior because we need
-     * to reconstruct any left-overs and clean them up, as well as work
-     * out which files are temporary. So for these cases we don't want the full
-     * transactional behavior, plus it's handy for the TransactionTidier.
-     */
-    final static class TransactionData implements AutoCloseable
-    {
-        private final OperationType opType;
-        private final UUID id;
-        private final File folder;
-        private final TransactionFile[] files;
-        private int folderDescriptor;
-        private boolean succeeded;
-
-        static TransactionData make(File logFile)
-        {
-            Matcher matcher = TransactionFile.REGEX.matcher(logFile.getName());
-            assert matcher.matches();
-
-            OperationType operationType = OperationType.fromFileName(matcher.group(1));
-            UUID id = UUID.fromString(matcher.group(2));
-
-            return new TransactionData(operationType, logFile.getParentFile(), id);
-        }
-
-        TransactionData(OperationType opType, File folder, UUID id)
-        {
-            this.opType = opType;
-            this.id = id;
-            this.folder = folder;
-            this.files = new TransactionFile[TransactionFile.Type.values().length];
-            for (TransactionFile.Type t : TransactionFile.Type.values())
-                this.files[t.idx] = new TransactionFile(t, this);
-
-            this.folderDescriptor = CLibrary.tryOpenDirectory(folder.getPath());
-            this.succeeded = !newLog().exists() && oldLog().exists();
-        }
-
-        public void succeeded(boolean succeeded)
-        {
-            this.succeeded = succeeded;
-        }
-
-        public void close()
-        {
-            if (folderDescriptor > 0)
-            {
-                CLibrary.tryCloseFD(folderDescriptor);
-                folderDescriptor = -1;
-            }
-        }
-
-        void crossReference()
-        {
-            newLog().add(oldLog().file.getPath());
-            oldLog().add(newLog().file.getPath());
-        }
-
-        void sync()
-        {
-            if (folderDescriptor > 0)
-                CLibrary.trySync(folderDescriptor);
-        }
-
-        TransactionFile newLog()
-        {
-            return files[TransactionFile.Type.NEW.idx];
-        }
-
-        TransactionFile oldLog()
-        {
-            return files[TransactionFile.Type.OLD.idx];
-        }
-
-        OperationType getType()
-        {
-            return opType;
-        }
-
-        UUID getId()
-        {
-            return id;
-        }
-
-        Throwable removeUnfinishedLeftovers(Throwable accumulate)
-        {
-            try
-            {
-                if (succeeded)
-                    oldLog().delete(true);
-                else
-                    newLog().delete(true);
-            }
-            catch (Throwable t)
-            {
-                accumulate = merge(accumulate, t);
-            }
-
-            return accumulate;
-        }
-
-        Set<File> getTemporaryFiles()
-        {
-            sync();
-
-            if (newLog().exists())
-                return newLog().getTrackedFiles();
-            else
-                return oldLog().getTrackedFiles();
-        }
-
-        String getFileName(TransactionFile.Type type)
-        {
-            String fileName = StringUtils.join(opType.fileName,
-                                               TransactionFile.SEP,
-                                               id.toString(),
-                                               TransactionFile.SEP,
-                                               type.txt,
-                                               TransactionFile.EXT);
-            return StringUtils.join(folder, File.separator, fileName);
-        }
-
-        String getParentFolder()
-        {
-            return folder.getParent();
-        }
-
-        static boolean isLogFile(String name)
-        {
-            return TransactionFile.REGEX.matcher(name).matches();
-        }
-    }
-
-    private final Tracker tracker;
-    private final TransactionData data;
-    private final Ref<TransactionLogs> selfRef;
-    // Deleting sstables is tricky because the mmapping might not have been finalized yet,
-    // and delete will fail (on Windows) until it is (we only force the unmapping on SUN VMs).
-    // Additionally, we need to make sure to delete the data file first, so on restart the others
-    // will be recognized as GCable.
-    private static final Queue<Runnable> failedDeletions = new ConcurrentLinkedQueue<>();
-    private static final Blocker blocker = new Blocker();
-
-    TransactionLogs(OperationType opType, CFMetaData metadata)
-    {
-        this(opType, metadata, null);
-    }
-
-    TransactionLogs(OperationType opType, CFMetaData metadata, Tracker tracker)
-    {
-        this(opType, new Directories(metadata), tracker);
-    }
-
-    TransactionLogs(OperationType opType, Directories directories, Tracker tracker)
-    {
-        this(opType, directories.getDirectoryForNewSSTables(), tracker);
-    }
-
-    TransactionLogs(OperationType opType, File folder, Tracker tracker)
-    {
-        this.tracker = tracker;
-        this.data = new TransactionData(opType,
-                                        Directories.getTransactionsDirectory(folder),
-                                        UUIDGen.getTimeUUID());
-        this.selfRef = new Ref<>(this, new TransactionTidier(data));
-
-        data.crossReference();
-        if (logger.isDebugEnabled())
-            logger.debug("Created transaction logs with id {}", data.id);
-    }
-
-    /**
-     * Track a reader as new.
-     **/
-    void trackNew(SSTable table)
-    {
-        if (!data.newLog().add(table))
-            throw new IllegalStateException(table + " is already tracked as new");
-
-        data.newLog().add(table);
-    }
-
-    /**
-     * Stop tracking a reader as new.
-     */
-    void untrackNew(SSTable table)
-    {
-        data.newLog().remove(table);
-    }
-
-    /**
-     * Schedule a reader for deletion as soon as it is fully unreferenced and the transaction
-     * has been committed.
-     */
-    SSTableTidier obsoleted(SSTableReader reader)
-    {
-        if (data.newLog().contains(reader))
-        {
-            if (data.oldLog().contains(reader))
-                throw new IllegalArgumentException();
-
-            return new SSTableTidier(reader, true, this);
-        }
-
-        if (!data.oldLog().add(reader))
-            throw new IllegalStateException();
-
-        if (tracker != null)
-            tracker.notifyDeleting(reader);
-
-        return new SSTableTidier(reader, false, this);
-    }
-
-    OperationType getType()
-    {
-        return data.getType();
-    }
-
-    UUID getId()
-    {
-        return data.getId();
-    }
-
-    @VisibleForTesting
-    String getDataFolder()
-    {
-        return data.getParentFolder();
-    }
-
-    @VisibleForTesting
-    String getLogsFolder()
-    {
-        return StringUtils.join(getDataFolder(), File.separator, Directories.TRANSACTIONS_SUBDIR);
-    }
-
-    @VisibleForTesting
-    TransactionData getData()
-    {
-        return data;
-    }
-
-    private static void delete(File file)
-    {
-        try
-        {
-            if (logger.isDebugEnabled())
-                logger.debug("Deleting {}", file);
-
-            Files.delete(file.toPath());
-        }
-        catch (NoSuchFileException e)
-        {
-            logger.error("Unable to delete {} as it does not exist", file);
-        }
-        catch (IOException e)
-        {
-            logger.error("Unable to delete {}", file, e);
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * The transaction tidier.
-     *
-     * When the transaction reference is fully released we try to delete all the obsolete files
-     * depending on the transaction result.
-     */
-    private static class TransactionTidier implements RefCounted.Tidy, Runnable
-    {
-        private final TransactionData data;
-
-        public TransactionTidier(TransactionData data)
-        {
-            this.data = data;
-        }
-
-        public void tidy() throws Exception
-        {
-            run();
-        }
-
-        public String name()
-        {
-            return data.id.toString();
-        }
-
-        public void run()
-        {
-            if (logger.isDebugEnabled())
-                logger.debug("Removing files for transaction {}", name());
-
-            Throwable err = data.removeUnfinishedLeftovers(null);
-
-            if (err != null)
-            {
-                logger.info("Failed deleting files for transaction {}, we'll retry after GC and on on server restart", name(), err);
-                failedDeletions.add(this);
-            }
-            else
-            {
-                if (logger.isDebugEnabled())
-                    logger.debug("Closing file transaction {}", name());
-                data.close();
-            }
-        }
-    }
-
-    static class Obsoletion
-    {
-        final SSTableReader reader;
-        final SSTableTidier tidier;
-
-        public Obsoletion(SSTableReader reader, SSTableTidier tidier)
-        {
-            this.reader = reader;
-            this.tidier = tidier;
-        }
-    }
-
-    /**
-     * The SSTableReader tidier. When a reader is fully released and no longer referenced
-     * by any one, we run this. It keeps a reference to the parent transaction and releases
-     * it when done, so that the final transaction cleanup can run when all obsolete readers
-     * are released.
-     */
-    public static class SSTableTidier implements Runnable
-    {
-        // must not retain a reference to the SSTableReader, else leak detection cannot kick in
-        private final Descriptor desc;
-        private final long sizeOnDisk;
-        private final Tracker tracker;
-        private final boolean wasNew;
-        private final Ref<TransactionLogs> parentRef;
-
-        public SSTableTidier(SSTableReader referent, boolean wasNew, TransactionLogs parent)
-        {
-            this.desc = referent.descriptor;
-            this.sizeOnDisk = referent.bytesOnDisk();
-            this.tracker = parent.tracker;
-            this.wasNew = wasNew;
-            this.parentRef = parent.selfRef.tryRef();
-        }
-
-        public void run()
-        {
-            blocker.ask();
-
-            SystemKeyspace.clearSSTableReadMeter(desc.ksname, desc.cfname, desc.generation);
-
-            try
-            {
-                // If we can't successfully delete the DATA component, set the task to be retried later: see TransactionTidier
-                File datafile = new File(desc.filenameFor(Component.DATA));
-
-                delete(datafile);
-                // let the remainder be cleaned up by delete
-                SSTable.delete(desc, SSTable.discoverComponentsFor(desc));
-            }
-            catch (Throwable t)
-            {
-                logger.error("Failed deletion for {}, we'll retry after GC and on server restart", desc);
-                failedDeletions.add(this);
-                return;
-            }
-
-            if (tracker != null && tracker.cfstore != null && !wasNew)
-                tracker.cfstore.metric.totalDiskSpaceUsed.dec(sizeOnDisk);
-
-            // release the referent to the parent so that the all transaction files can be released
-            parentRef.release();
-        }
-
-        public void abort()
-        {
-            parentRef.release();
-        }
-    }
-
-    /**
-     * Retry all deletions that failed the first time around (presumably b/c the sstable was still mmap'd.)
-     * Useful because there are times when we know GC has been invoked; also exposed as an mbean.
-     */
-    public static void rescheduleFailedDeletions()
-    {
-        Runnable task;
-        while ( null != (task = failedDeletions.poll()))
-            ScheduledExecutors.nonPeriodicTasks.submit(task);
-    }
-
-    /**
-     * Deletions run on the nonPeriodicTasks executor, (both failedDeletions or global tidiers in SSTableReader)
-     * so by scheduling a new empty task and waiting for it we ensure any prior deletion has completed.
-     */
-    public static void waitForDeletions()
-    {
-        FBUtilities.waitOnFuture(ScheduledExecutors.nonPeriodicTasks.schedule(() -> {
-        }, 0, TimeUnit.MILLISECONDS));
-    }
-
-    @VisibleForTesting
-    public static void pauseDeletions(boolean stop)
-    {
-        blocker.block(stop);
-    }
-
-    private Throwable complete(Throwable accumulate)
-    {
-        try
-        {
-            try
-            {
-                if (data.succeeded)
-                    data.newLog().delete(false);
-                else
-                    data.oldLog().delete(false);
-            }
-            catch (Throwable t)
-            {
-                accumulate = merge(accumulate, t);
-            }
-
-            accumulate = selfRef.ensureReleased(accumulate);
-            return accumulate;
-        }
-        catch (Throwable t)
-        {
-            logger.error("Failed to complete file transaction {}", getId(), t);
-            return Throwables.merge(accumulate, t);
-        }
-    }
-
-    protected Throwable doCommit(Throwable accumulate)
-    {
-        data.succeeded(true);
-        return complete(accumulate);
-    }
-
-    protected Throwable doAbort(Throwable accumulate)
-    {
-        data.succeeded(false);
-        return complete(accumulate);
-    }
-
-    protected void doPrepare() { }
-
-    /**
-     * Called on startup to scan existing folders for any unfinished leftovers of
-     * operations that were ongoing when the process exited.
-     *
-     * We check if the new transaction file exists first, and if so we clean it up
-     * along with its contents, which includes the old file, else if only the old file exists
-     * it means the operation has completed and we only cleanup the old file with its contents.
-     */
-    static void removeUnfinishedLeftovers(CFMetaData metadata)
-    {
-        Throwable accumulate = null;
-        Set<UUID> ids = new HashSet<>();
-
-        for (File dir : getFolders(metadata, null))
-        {
-            File[] logs = dir.listFiles((dir1, name) -> {
-                return TransactionData.isLogFile(name);
-            });
-
-            for (File log : logs)
-            {
-                try (TransactionData data = TransactionData.make(log))
-                {
-                    // we need to check this because there are potentially 2 log files per operation
-                    if (ids.contains(data.id))
-                        continue;
-
-                    ids.add(data.id);
-                    accumulate = data.removeUnfinishedLeftovers(accumulate);
-                }
-            }
-        }
-
-        if (accumulate != null)
-            logger.error("Failed to remove unfinished transaction leftovers", accumulate);
-    }
-
-    /**
-     * Return a set of files that are temporary, that is they are involved with
-     * a transaction that hasn't completed yet.
-     *
-     * Only return the files that exist and that are located in the folder
-     * specified as a parameter or its sub-folders.
-     */
-    static Set<File> getTemporaryFiles(CFMetaData metadata, File folder)
-    {
-        Set<File> ret = new HashSet<>();
-        Set<UUID> ids = new HashSet<>();
-
-        for (File dir : getFolders(metadata, folder))
-        {
-            File[] logs = dir.listFiles((dir1, name) -> {
-                return TransactionData.isLogFile(name);
-            });
-
-            for (File log : logs)
-            {
-                try(TransactionData data = TransactionData.make(log))
-                {
-                    // we need to check this because there are potentially 2 log files per transaction
-                    if (ids.contains(data.id))
-                        continue;
-
-                    ids.add(data.id);
-                    ret.addAll(data.getTemporaryFiles()
-                                   .stream()
-                                   .filter(file -> FileUtils.isContained(folder, file))
-                                   .collect(Collectors.toSet()));
-                }
-            }
-        }
-
-        return ret;
-    }
-
-    /**
-     * Return the transaction log files that currently exist for this table.
-     */
-    static Set<File> getLogFiles(CFMetaData metadata)
-    {
-        Set<File> ret = new HashSet<>();
-        for (File dir : getFolders(metadata, null))
-            ret.addAll(Arrays.asList(dir.listFiles((dir1, name) -> {
-                return TransactionData.isLogFile(name);
-            })));
-
-        return ret;
-    }
-
-    /**
-     * A utility method to work out the existing transaction sub-folders
-     * either for a table, or a specific parent folder, or both.
-     */
-    private static List<File> getFolders(CFMetaData metadata, File folder)
-    {
-        List<File> ret = new ArrayList<>();
-        if (metadata != null)
-        {
-            Directories directories = new Directories(metadata);
-            ret.addAll(directories.getExistingDirectories(Directories.TRANSACTIONS_SUBDIR));
-        }
-
-        if (folder != null)
-        {
-            File opDir = Directories.getExistingDirectory(folder, Directories.TRANSACTIONS_SUBDIR);
-            if (opDir != null)
-                ret.add(opDir);
-        }
-
-        return ret;
-    }
-}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
index 20c3962..3286522 100644
--- a/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
+++ b/src/java/org/apache/cassandra/io/sstable/SSTableLoader.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.io.sstable;
 
 import java.io.File;
-import java.io.FilenameFilter;
 import java.io.IOException;
 import java.net.InetAddress;
 import java.util.*;
@@ -27,16 +26,13 @@ import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 
 import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
 import org.apache.cassandra.dht.Token;
-import org.apache.cassandra.exceptions.ConfigurationException;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.service.ActiveRepairService;
 import org.apache.cassandra.streaming.*;
-import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.OutputHandler;
 import org.apache.cassandra.utils.Pair;
 
@@ -77,89 +73,79 @@ public class SSTableLoader implements StreamEventHandler
     {
         outputHandler.output("Opening sstables and calculating sections to stream");
 
-        directory.list(new FilenameFilter()
-        {
-            final Map<File, Set<File>> allTemporaryFiles = new HashMap<>();
-            public boolean accept(File dir, String name)
-            {
-                File file = new File(dir, name);
-
-                if (file.isDirectory())
-                    return false;
-
-                Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
-                Descriptor desc = p == null ? null : p.left;
-                if (p == null || !p.right.equals(Component.DATA))
-                    return false;
-
-                if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists())
-                {
-                    outputHandler.output(String.format("Skipping file %s because index is missing", name));
-                    return false;
-                }
-
-                CFMetaData metadata = client.getTableMetadata(desc.cfname);
-                if (metadata == null)
-                {
-                    outputHandler.output(String.format("Skipping file %s: table %s.%s doesn't exist", name, keyspace, desc.cfname));
-                    return false;
-                }
-
-                Set<File> temporaryFiles = allTemporaryFiles.get(dir);
-                if (temporaryFiles == null)
-                {
-                    temporaryFiles = LifecycleTransaction.getTemporaryFiles(metadata, dir);
-                    allTemporaryFiles.put(dir, temporaryFiles);
-                }
-
-                if (temporaryFiles.contains(file))
-                {
-                    outputHandler.output(String.format("Skipping temporary file %s", name));
-                    return false;
-                }
-
-                Set<Component> components = new HashSet<>();
-                components.add(Component.DATA);
-                components.add(Component.PRIMARY_INDEX);
-                if (new File(desc.filenameFor(Component.SUMMARY)).exists())
-                    components.add(Component.SUMMARY);
-                if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
-                    components.add(Component.COMPRESSION_INFO);
-                if (new File(desc.filenameFor(Component.STATS)).exists())
-                    components.add(Component.STATS);
-
-                try
-                {
-                    // To conserve memory, open SSTableReaders without bloom filters and discard
-                    // the index summary after calculating the file sections to stream and the estimated
-                    // number of keys for each endpoint. See CASSANDRA-5555 for details.
-                    SSTableReader sstable = SSTableReader.openForBatch(desc, components, metadata);
-                    sstables.add(sstable);
-
-                    // calculate the sstable sections to stream as well as the estimated number of
-                    // keys per host
-                    for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : ranges.entrySet())
-                    {
-                        InetAddress endpoint = entry.getKey();
-                        Collection<Range<Token>> tokenRanges = entry.getValue();
-
-                        List<Pair<Long, Long>> sstableSections = sstable.getPositionsForRanges(tokenRanges);
-                        long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges);
-                        Ref<SSTableReader> ref = sstable.ref();
-                        StreamSession.SSTableStreamingSections details = new StreamSession.SSTableStreamingSections(ref, sstableSections, estimatedKeys, ActiveRepairService.UNREPAIRED_SSTABLE);
-                        streamingDetails.put(endpoint, details);
-                    }
-
-                    // to conserve heap space when bulk loading
-                    sstable.releaseSummary();
-                }
-                catch (IOException e)
-                {
-                    outputHandler.output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
-                }
-                return false;
-            }
-        });
+        LifecycleTransaction.getFiles(directory.toPath(),
+                                      (file, type) ->
+                                      {
+                                          File dir = file.getParentFile();
+                                          String name = file.getName();
+
+                                          if (type != Directories.FileType.FINAL)
+                                          {
+                                              outputHandler.output(String.format("Skipping temporary file %s", name));
+                                              return false;
+                                          }
+
+                                          Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(dir, name);
+                                          Descriptor desc = p == null ? null : p.left;
+                                          if (p == null || !p.right.equals(Component.DATA))
+                                              return false;
+
+                                          if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists())
+                                          {
+                                              outputHandler.output(String.format("Skipping file %s because index is missing", name));
+                                              return false;
+                                          }
+
+                                          CFMetaData metadata = client.getTableMetadata(desc.cfname);
+                                          if (metadata == null)
+                                          {
+                                              outputHandler.output(String.format("Skipping file %s: table %s.%s doesn't exist", name, keyspace, desc.cfname));
+                                              return false;
+                                          }
+
+                                          Set<Component> components = new HashSet<>();
+                                          components.add(Component.DATA);
+                                          components.add(Component.PRIMARY_INDEX);
+                                          if (new File(desc.filenameFor(Component.SUMMARY)).exists())
+                                              components.add(Component.SUMMARY);
+                                          if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
+                                              components.add(Component.COMPRESSION_INFO);
+                                          if (new File(desc.filenameFor(Component.STATS)).exists())
+                                              components.add(Component.STATS);
+
+                                          try
+                                          {
+                                              // To conserve memory, open SSTableReaders without bloom filters and discard
+                                              // the index summary after calculating the file sections to stream and the estimated
+                                              // number of keys for each endpoint. See CASSANDRA-5555 for details.
+                                              SSTableReader sstable = SSTableReader.openForBatch(desc, components, metadata);
+                                              sstables.add(sstable);
+
+                                              // calculate the sstable sections to stream as well as the estimated number of
+                                              // keys per host
+                                              for (Map.Entry<InetAddress, Collection<Range<Token>>> entry : ranges.entrySet())
+                                              {
+                                                  InetAddress endpoint = entry.getKey();
+                                                  Collection<Range<Token>> tokenRanges = entry.getValue();
+
+                                                  List<Pair<Long, Long>> sstableSections = sstable.getPositionsForRanges(tokenRanges);
+                                                  long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges);
+                                                  Ref<SSTableReader> ref = sstable.ref();
+                                                  StreamSession.SSTableStreamingSections details = new StreamSession.SSTableStreamingSections(ref, sstableSections, estimatedKeys, ActiveRepairService.UNREPAIRED_SSTABLE);
+                                                  streamingDetails.put(endpoint, details);
+                                              }
+
+                                              // to conserve heap space when bulk loading
+                                              sstable.releaseSummary();
+                                          }
+                                          catch (IOException e)
+                                          {
+                                              outputHandler.output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
+                                          }
+                                          return false;
+                                      },
+                                      Directories.OnTxnErr.IGNORE);
+
         return sstables;
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
index d5c192e..5502669 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java
@@ -45,7 +45,7 @@ import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.db.filter.ColumnFilter;
 import org.apache.cassandra.db.index.SecondaryIndex;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.db.rows.*;
 import org.apache.cassandra.dht.*;
 import org.apache.cassandra.io.FSError;
@@ -1647,7 +1647,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
      * @return true if the this is the first time the file was marked obsolete.  Calling this
      * multiple times is usually buggy (see exceptions in Tracker.unmarkCompacting and removeOldSSTablesSize).
      */
-    public void markObsolete(TransactionLogs.SSTableTidier tidier)
+    public void markObsolete(TransactionLog.SSTableTidier tidier)
     {
         if (logger.isDebugEnabled())
             logger.debug("Marking {} compacted", getFilename());
@@ -1903,22 +1903,6 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
              : (sstableMetadata.totalRows == 0 ? 0 : (int)(sstableMetadata.totalColumnsSet / sstableMetadata.totalRows));
     }
 
-    public Set<Integer> getAncestors()
-    {
-        try
-        {
-            CompactionMetadata compactionMetadata = (CompactionMetadata) descriptor.getMetadataSerializer().deserialize(descriptor, MetadataType.COMPACTION);
-            if (compactionMetadata != null)
-                return compactionMetadata.ancestors;
-            return Collections.emptySet();
-        }
-        catch (IOException e)
-        {
-            SSTableReader.logOpenException(descriptor, e);
-            return Collections.emptySet();
-        }
-    }
-
     public int getSSTableLevel()
     {
         return sstableMetadata.sstableLevel;
@@ -2191,7 +2175,7 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
         // sstable have been released
         private ScheduledFuture readMeterSyncFuture;
         // shared state managing if the logical sstable has been compacted; this is used in cleanup
-        private volatile TransactionLogs.SSTableTidier obsoletion;
+        private volatile TransactionLog.SSTableTidier obsoletion;
 
         GlobalTidy(final SSTableReader reader)
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/format/Version.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/Version.java b/src/java/org/apache/cassandra/io/sstable/format/Version.java
index 9ef0b43..4fcf055 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/Version.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/Version.java
@@ -66,6 +66,8 @@ public abstract class Version
 
     public abstract boolean hasOldBfHashOrder();
 
+    public abstract boolean hasCompactionAncestors();
+
     public String getVersion()
     {
         return version;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
index 6df4b1e..d65710e 100644
--- a/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
+++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigFormat.java
@@ -140,6 +140,11 @@ public class BigFormat implements SSTableFormat
          */
         private final boolean hasOldBfHashOrder;
 
+        /**
+         * CASSANDRA-7066: compaction ancerstors are no longer used and have been removed.
+         */
+        private final boolean hasCompactionAncestors;
+
         BigVersion(String version)
         {
             super(instance, version);
@@ -166,6 +171,7 @@ public class BigFormat implements SSTableFormat
             newFileName = version.compareTo("la") >= 0;
 
             hasOldBfHashOrder = version.compareTo("ma") < 0;
+            hasCompactionAncestors = version.compareTo("ma") < 0;
             storeRows = version.compareTo("ma") >= 0;
             correspondingMessagingVersion = storeRows
                                           ? MessagingService.VERSION_30
@@ -221,6 +227,12 @@ public class BigFormat implements SSTableFormat
         }
 
         @Override
+        public boolean hasCompactionAncestors()
+        {
+            return hasCompactionAncestors;
+        }
+
+        @Override
         public boolean hasNewFileName()
         {
             return newFileName;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
index 29cbe5b..fa94ead 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/CompactionMetadata.java
@@ -18,8 +18,6 @@
 package org.apache.cassandra.io.sstable.metadata;
 
 import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
 
 import com.clearspring.analytics.stream.cardinality.HyperLogLogPlus;
 import com.clearspring.analytics.stream.cardinality.ICardinality;
@@ -39,13 +37,10 @@ public class CompactionMetadata extends MetadataComponent
 {
     public static final IMetadataComponentSerializer serializer = new CompactionMetadataSerializer();
 
-    public final Set<Integer> ancestors;
-
     public final ICardinality cardinalityEstimator;
 
-    public CompactionMetadata(Set<Integer> ancestors, ICardinality cardinalityEstimator)
+    public CompactionMetadata(ICardinality cardinalityEstimator)
     {
-        this.ancestors = ancestors;
         this.cardinalityEstimator = cardinalityEstimator;
     }
 
@@ -57,48 +52,46 @@ public class CompactionMetadata extends MetadataComponent
     @Override
     public boolean equals(Object o)
     {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
+        if (this == o)
+            return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
 
-        CompactionMetadata that = (CompactionMetadata) o;
-        return ancestors == null ? that.ancestors == null : ancestors.equals(that.ancestors);
+        // keeping equals and hashCode as all classes inheriting from MetadataComponent
+        // implement them but we have really nothing to compare
+        return true;
     }
 
     @Override
     public int hashCode()
     {
-        return ancestors != null ? ancestors.hashCode() : 0;
+        // see comment in equals
+        return 31;
     }
 
     public static class CompactionMetadataSerializer implements IMetadataComponentSerializer<CompactionMetadata>
     {
         public int serializedSize(CompactionMetadata component) throws IOException
         {
-            int size = 0;
-            size += TypeSizes.sizeof(component.ancestors.size());
-            for (int g : component.ancestors)
-                size += TypeSizes.sizeof(g);
             byte[] serializedCardinality = component.cardinalityEstimator.getBytes();
-            size += TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length;
-            return size;
+            return TypeSizes.sizeof(serializedCardinality.length) + serializedCardinality.length;
         }
 
         public void serialize(CompactionMetadata component, DataOutputPlus out) throws IOException
         {
-            out.writeInt(component.ancestors.size());
-            for (int g : component.ancestors)
-                out.writeInt(g);
             ByteBufferUtil.writeWithLength(component.cardinalityEstimator.getBytes(), out);
         }
 
         public CompactionMetadata deserialize(Version version, DataInputPlus in) throws IOException
         {
-            int nbAncestors = in.readInt();
-            Set<Integer> ancestors = new HashSet<>(nbAncestors);
-            for (int i = 0; i < nbAncestors; i++)
-                ancestors.add(in.readInt());
+            if (version.hasCompactionAncestors())
+            { // skip ancestors
+                int nbAncestors = in.readInt();
+                in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
+            }
             ICardinality cardinality = HyperLogLogPlus.Builder.build(ByteBufferUtil.readBytes(in, in.readInt()));
-            return new CompactionMetadata(ancestors, cardinality);
+            return new CompactionMetadata(cardinality);
         }
     }
 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
index 433c31a..53ba922 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/LegacyMetadataSerializer.java
@@ -23,6 +23,7 @@ import java.util.*;
 
 import com.google.common.collect.Maps;
 
+import org.apache.cassandra.db.TypeSizes;
 import org.apache.cassandra.db.commitlog.ReplayPosition;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
@@ -60,9 +61,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
         out.writeDouble(validation.bloomFilterFPChance);
         out.writeDouble(stats.compressionRatio);
         out.writeUTF(validation.partitioner);
-        out.writeInt(compaction.ancestors.size());
-        for (Integer g : compaction.ancestors)
-            out.writeInt(g);
+        out.writeInt(0); // compaction ancestors
         StreamingHistogram.serializer.serialize(stats.estimatedTombstoneDropTime, out);
         out.writeInt(stats.sstableLevel);
         out.writeInt(stats.minClusteringValues.size());
@@ -99,10 +98,8 @@ public class LegacyMetadataSerializer extends MetadataSerializer
                 double bloomFilterFPChance = in.readDouble();
                 double compressionRatio = in.readDouble();
                 String partitioner = in.readUTF();
-                int nbAncestors = in.readInt();
-                Set<Integer> ancestors = new HashSet<>(nbAncestors);
-                for (int i = 0; i < nbAncestors; i++)
-                    ancestors.add(in.readInt());
+                int nbAncestors = in.readInt(); //skip compaction ancestors
+                in.skipBytes(nbAncestors * TypeSizes.sizeof(nbAncestors));
                 StreamingHistogram tombstoneHistogram = StreamingHistogram.serializer.deserialize(in);
                 int sstableLevel = 0;
                 if (in.available() > 0)
@@ -143,7 +140,7 @@ public class LegacyMetadataSerializer extends MetadataSerializer
                                                      -1));
                 if (types.contains(MetadataType.COMPACTION))
                     components.put(MetadataType.COMPACTION,
-                                   new CompactionMetadata(ancestors, null));
+                                   new CompactionMetadata(null));
             }
         }
         return components;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
index 9b06b53..1c93f58 100644
--- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
+++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataCollector.java
@@ -94,7 +94,6 @@ public class MetadataCollector implements PartitionStatisticsCollector
     protected final MinMaxIntTracker localDeletionTimeTracker = new MinMaxIntTracker(Cell.NO_DELETION_TIME, Cell.NO_DELETION_TIME);
     protected final MinMaxIntTracker ttlTracker = new MinMaxIntTracker(Cell.NO_TTL, Cell.NO_TTL);
     protected double compressionRatio = NO_COMPRESSION_RATIO;
-    protected Set<Integer> ancestors = new HashSet<>();
     protected StreamingHistogram estimatedTombstoneDropTime = defaultTombstoneDropTimeHistogram();
     protected int sstableLevel;
     protected ByteBuffer[] minClusteringValues;
@@ -120,29 +119,12 @@ public class MetadataCollector implements PartitionStatisticsCollector
         this.maxClusteringValues = new ByteBuffer[comparator.size()];
     }
 
-    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level, boolean skipAncestors)
+    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
     {
         this(comparator);
 
         replayPosition(ReplayPosition.getReplayPosition(sstables));
         sstableLevel(level);
-        // Get the max timestamp of the precompacted sstables
-        // and adds generation of live ancestors
-        if (!skipAncestors)
-        {
-            for (SSTableReader sstable : sstables)
-            {
-                addAncestor(sstable.descriptor.generation);
-                for (Integer i : sstable.getAncestors())
-                    if (new File(sstable.descriptor.withGeneration(i).filenameFor(Component.DATA)).exists())
-                        addAncestor(i);
-            }
-        }
-    }
-
-    public MetadataCollector(Iterable<SSTableReader> sstables, ClusteringComparator comparator, int level)
-    {
-        this(sstables, comparator, level, false);
     }
 
     public MetadataCollector addKey(ByteBuffer key)
@@ -237,12 +219,6 @@ public class MetadataCollector implements PartitionStatisticsCollector
         return this;
     }
 
-    public MetadataCollector addAncestor(int generation)
-    {
-        this.ancestors.add(generation);
-        return this;
-    }
-
     public MetadataCollector sstableLevel(int sstableLevel)
     {
         this.sstableLevel = sstableLevel;
@@ -313,7 +289,7 @@ public class MetadataCollector implements PartitionStatisticsCollector
                                                              repairedAt,
                                                              totalColumnsSet,
                                                              totalRows));
-        components.put(MetadataType.COMPACTION, new CompactionMetadata(ancestors, cardinality));
+        components.put(MetadataType.COMPACTION, new CompactionMetadata(cardinality));
         components.put(MetadataType.HEADER, header.toComponent());
         return components;
     }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/io/util/FileUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/io/util/FileUtils.java b/src/java/org/apache/cassandra/io/util/FileUtils.java
index c3de1db..0ba2307 100644
--- a/src/java/org/apache/cassandra/io/util/FileUtils.java
+++ b/src/java/org/apache/cassandra/io/util/FileUtils.java
@@ -616,11 +616,6 @@ public class FileUtils
         {
             return Files.readAllLines(file.toPath(), Charset.forName("utf-8"));
         }
-        catch (NoSuchFileException ex)
-        {
-            logger.warn("Tried to read non existing file: {}", file);
-            return Collections.emptyList();
-        }
         catch (IOException ex)
         {
             throw new RuntimeException(ex);

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/service/GCInspector.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/GCInspector.java b/src/java/org/apache/cassandra/service/GCInspector.java
index fc7ff3d..252d1c3 100644
--- a/src/java/org/apache/cassandra/service/GCInspector.java
+++ b/src/java/org/apache/cassandra/service/GCInspector.java
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
 import com.sun.management.GarbageCollectionNotificationInfo;
 import com.sun.management.GcInfo;
 
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.utils.StatusLogger;
 
 public class GCInspector implements NotificationListener, GCInspectorMXBean
@@ -284,7 +284,7 @@ public class GCInspector implements NotificationListener, GCInspectorMXBean
 
             // if we just finished an old gen collection and we're still using a lot of memory, try to reduce the pressure
             if (gcState.assumeGCIsOldGen)
-                TransactionLogs.rescheduleFailedDeletions();
+                TransactionLog.rescheduleFailedDeletions();
         }
     }
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/service/StartupChecks.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StartupChecks.java b/src/java/org/apache/cassandra/service/StartupChecks.java
index fee9789..9ffef96 100644
--- a/src/java/org/apache/cassandra/service/StartupChecks.java
+++ b/src/java/org/apache/cassandra/service/StartupChecks.java
@@ -237,8 +237,7 @@ public class StartupChecks
                 {
                     String name = dir.getFileName().toString();
                     return (name.equals(Directories.SNAPSHOT_SUBDIR)
-                            || name.equals(Directories.BACKUPS_SUBDIR)
-                            || name.equals(Directories.TRANSACTIONS_SUBDIR))
+                            || name.equals(Directories.BACKUPS_SUBDIR))
                            ? FileVisitResult.SKIP_SUBTREE
                            : FileVisitResult.CONTINUE;
                 }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/service/StorageService.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/service/StorageService.java b/src/java/org/apache/cassandra/service/StorageService.java
index 959f7e3..5966e49 100644
--- a/src/java/org/apache/cassandra/service/StorageService.java
+++ b/src/java/org/apache/cassandra/service/StorageService.java
@@ -70,7 +70,7 @@ import org.apache.cassandra.config.*;
 import org.apache.cassandra.db.*;
 import org.apache.cassandra.db.commitlog.CommitLog;
 import org.apache.cassandra.db.compaction.CompactionManager;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.dht.BootStrapper;
 import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.Range;
@@ -4241,7 +4241,7 @@ public class StorageService extends NotificationBroadcasterSupport implements IE
 
     public void rescheduleFailedDeletions()
     {
-        TransactionLogs.rescheduleFailedDeletions();
+        TransactionLog.rescheduleFailedDeletions();
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
index bc17750..c8587d8 100644
--- a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
@@ -34,6 +34,7 @@ import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.db.ColumnFamilyStore;
 import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
@@ -68,7 +69,7 @@ public class SSTableExpiredBlockers
 
         Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
-        Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+        Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
         Set<SSTableReader> sstables = new HashSet<>();
         for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
index d73a325..cb3cc5c 100644
--- a/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
+++ b/src/java/org/apache/cassandra/tools/SSTableLevelResetter.java
@@ -23,6 +23,7 @@ import java.util.Set;
 
 import org.apache.cassandra.config.Schema;
 import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.io.sstable.Component;
 import org.apache.cassandra.io.sstable.Descriptor;
@@ -75,7 +76,7 @@ public class SSTableLevelResetter
             Keyspace keyspace = Keyspace.openWithoutSSTables(keyspaceName);
             ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(columnfamily);
             boolean foundSSTable = false;
-            for (Map.Entry<Descriptor, Set<Component>> sstable : cfs.directories.sstableLister().list().entrySet())
+            for (Map.Entry<Descriptor, Set<Component>> sstable : cfs.directories.sstableLister(Directories.OnTxnErr.THROW).list().entrySet())
             {
                 if (sstable.getValue().contains(Component.STATS))
                 {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
index a644f49..7764bbf 100644
--- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
+++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java
@@ -78,9 +78,7 @@ public class SSTableMetadataViewer
                 }
                 if (compaction != null)
                 {
-                    out.printf("Ancestors: %s%n", compaction.ancestors.toString());
                     out.printf("Estimated cardinality: %s%n", compaction.cardinalityEstimator.cardinality());
-
                 }
             }
             else

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
index f7e477f..95f516a 100644
--- a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
+++ b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
@@ -95,7 +95,7 @@ public class SSTableOfflineRelevel
 
         Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
         ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
-        Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+        Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
         Set<SSTableReader> sstables = new HashSet<>();
         for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
         {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneLister.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneLister.java b/src/java/org/apache/cassandra/tools/StandaloneLister.java
deleted file mode 100644
index 71575c7..0000000
--- a/src/java/org/apache/cassandra/tools/StandaloneLister.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.cassandra.tools;
-
-import org.apache.cassandra.config.CFMetaData;
-import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.Directories;
-import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.utils.OutputHandler;
-import org.apache.commons.cli.*;
-
-import java.io.File;
-
-import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
-
-public class StandaloneLister
-{
-    private static final String TOOL_NAME = "sstablelister";
-    private static final String TYPE_OPTION  = "type";
-    private static final String OP_LOG_OPTION  = "oplog";
-    private static final String VERBOSE_OPTION  = "verbose";
-    private static final String DEBUG_OPTION  = "debug";
-    private static final String HELP_OPTION  = "help";
-
-    public static void main(String args[])
-    {
-        Options options = Options.parseArgs(args);
-        try
-        {
-            // load keyspace descriptions.
-            Schema.instance.loadFromDisk(false);
-
-            CFMetaData metadata = Schema.instance.getCFMetaData(options.keyspaceName, options.cfName);
-            if (metadata == null)
-                throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
-                                                                 options.keyspaceName,
-                                                                 options.cfName));
-
-            OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
-
-            Directories directories = new Directories(metadata);
-            Directories.SSTableLister lister = directories.sstableLister();
-
-            if (options.type == Options.FileType.FINAL)
-                lister.skipTemporary(true);
-            else if (options.type == Options.FileType.TMP)
-                lister.onlyTemporary(true);
-
-            for (File file : lister.listFiles())
-               handler.output(file.getCanonicalPath());
-
-            if (options.oplogs)
-            {
-                for (File file : LifecycleTransaction.getLogFiles(metadata))
-                {
-                    handler.output(file.getCanonicalPath());
-                }
-            }
-
-            System.exit(0);
-        }
-        catch (Exception e)
-        {
-            System.err.println(e.getMessage());
-            if (options.debug)
-                e.printStackTrace(System.err);
-            System.exit(1);
-        }
-    }
-
-    private static class Options
-    {
-        public enum FileType
-        {
-            ALL("all", "list all files, final or temporary"),
-            TMP("tmp", "list temporary files only"),
-            FINAL("final", "list final files only");
-
-            public String option;
-            public String descr;
-            FileType(String option, String descr)
-            {
-                this.option = option;
-                this.descr = descr;
-            }
-
-            static FileType fromOption(String option)
-            {
-                for (FileType fileType : FileType.values())
-                {
-                    if (fileType.option.equals(option))
-                        return fileType;
-                }
-
-                return FileType.ALL;
-            }
-
-            static String descr()
-            {
-                StringBuilder str = new StringBuilder();
-                for (FileType fileType : FileType.values())
-                {
-                    str.append(fileType.option);
-                    str.append(" (");
-                    str.append(fileType.descr);
-                    str.append("), ");
-                }
-                return str.toString();
-            }
-        }
-
-        public final String keyspaceName;
-        public final String cfName;
-
-        public boolean debug;
-        public boolean verbose;
-        public boolean oplogs;
-        public FileType type;
-
-        private Options(String keyspaceName, String cfName)
-        {
-            this.keyspaceName = keyspaceName;
-            this.cfName = cfName;
-        }
-
-        public static Options parseArgs(String cmdArgs[])
-        {
-            CommandLineParser parser = new GnuParser();
-            CmdLineOptions options = getCmdLineOptions();
-            try
-            {
-                CommandLine cmd = parser.parse(options, cmdArgs, false);
-
-                if (cmd.hasOption(HELP_OPTION))
-                {
-                    printUsage(options);
-                    System.exit(0);
-                }
-
-                String[] args = cmd.getArgs();
-                if (args.length != 2)
-                {
-                    String msg = args.length < 2 ? "Missing arguments" : "Too many arguments";
-                    System.err.println(msg);
-                    printUsage(options);
-                    System.exit(1);
-                }
-
-                String keyspaceName = args[0];
-                String cfName = args[1];
-
-                Options opts = new Options(keyspaceName, cfName);
-
-                opts.debug = cmd.hasOption(DEBUG_OPTION);
-                opts.verbose = cmd.hasOption(VERBOSE_OPTION);
-                opts.type = FileType.fromOption(cmd.getOptionValue(TYPE_OPTION));
-                opts.oplogs = cmd.hasOption(OP_LOG_OPTION);
-
-                return opts;
-            }
-            catch (ParseException e)
-            {
-                errorMsg(e.getMessage(), options);
-                return null;
-            }
-        }
-
-        private static void errorMsg(String msg, CmdLineOptions options)
-        {
-            System.err.println(msg);
-            printUsage(options);
-            System.exit(1);
-        }
-
-        private static CmdLineOptions getCmdLineOptions()
-        {
-            CmdLineOptions options = new CmdLineOptions();
-            options.addOption("d", DEBUG_OPTION, "display stack traces");
-            options.addOption("h", HELP_OPTION, "display this help message");
-            options.addOption("o", OP_LOG_OPTION, "include operation logs");
-            options.addOption("t", TYPE_OPTION, true, FileType.descr());
-            options.addOption("v", VERBOSE_OPTION, "verbose output");
-
-            return options;
-        }
-
-        public static void printUsage(CmdLineOptions options)
-        {
-            String usage = String.format("%s [options] <keyspace> <column_family>", TOOL_NAME);
-            StringBuilder header = new StringBuilder();
-            header.append("--\n");
-            header.append("List sstable files for the provided table." );
-            header.append("\n--\n");
-            header.append("Options are:");
-            new HelpFormatter().printHelp(usage, header.toString(), options, "");
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java b/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
new file mode 100644
index 0000000..6e2be1d
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/StandaloneSSTableUtil.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.cassandra.tools;
+
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
+import org.apache.cassandra.utils.OutputHandler;
+import org.apache.commons.cli.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.function.BiFunction;
+
+import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
+
+public class StandaloneSSTableUtil
+{
+    private static final String TOOL_NAME = "sstableutil";
+    private static final String TYPE_OPTION  = "type";
+    private static final String OP_LOG_OPTION  = "oplog";
+    private static final String VERBOSE_OPTION  = "verbose";
+    private static final String DEBUG_OPTION  = "debug";
+    private static final String HELP_OPTION  = "help";
+    private static final String CLEANUP_OPTION = "cleanup";
+
+    public static void main(String args[])
+    {
+        Options options = Options.parseArgs(args);
+        try
+        {
+            // load keyspace descriptions.
+            Schema.instance.loadFromDisk(false);
+
+            CFMetaData metadata = Schema.instance.getCFMetaData(options.keyspaceName, options.cfName);
+            if (metadata == null)
+                throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
+                                                                 options.keyspaceName,
+                                                                 options.cfName));
+
+            OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
+
+            if (options.cleanup)
+            {
+                handler.output("Cleanuping up...");
+                LifecycleTransaction.removeUnfinishedLeftovers(metadata);
+            }
+            else
+            {
+                handler.output("Listing files...");
+                listFiles(options, metadata, handler);
+            }
+
+            System.exit(0);
+        }
+        catch (Exception e)
+        {
+            System.err.println(e.getMessage());
+            if (options.debug)
+                e.printStackTrace(System.err);
+            System.exit(1);
+        }
+    }
+
+    private static void listFiles(Options options, CFMetaData metadata, OutputHandler handler) throws IOException
+    {
+        Directories directories = new Directories(metadata);
+
+        for (File dir : directories.getCFDirectories())
+        {
+            for (File file : LifecycleTransaction.getFiles(dir.toPath(), getFilter(options), Directories.OnTxnErr.THROW))
+                handler.output(file.getCanonicalPath());
+        }
+    }
+
+    private static BiFunction<File, Directories.FileType, Boolean> getFilter(Options options)
+    {
+        return (file, type) ->
+        {
+            switch(type)
+            {
+                case FINAL:
+                    return options.type != Options.FileType.TMP;
+                case TEMPORARY:
+                    return options.type != Options.FileType.FINAL;
+                case TXN_LOG:
+                    return options.oplogs;
+                default:
+                    throw new AssertionError();
+            }
+        };
+    }
+
+    private static class Options
+    {
+        public enum FileType
+        {
+            ALL("all", "list all files, final or temporary"),
+            TMP("tmp", "list temporary files only"),
+            FINAL("final", "list final files only");
+
+            public String option;
+            public String descr;
+            FileType(String option, String descr)
+            {
+                this.option = option;
+                this.descr = descr;
+            }
+
+            static FileType fromOption(String option)
+            {
+                for (FileType fileType : FileType.values())
+                {
+                    if (fileType.option.equals(option))
+                        return fileType;
+                }
+
+                return FileType.ALL;
+            }
+
+            static String descr()
+            {
+                StringBuilder str = new StringBuilder();
+                for (FileType fileType : FileType.values())
+                {
+                    str.append(fileType.option);
+                    str.append(" (");
+                    str.append(fileType.descr);
+                    str.append("), ");
+                }
+                return str.toString();
+            }
+        }
+
+        public final String keyspaceName;
+        public final String cfName;
+
+        public boolean debug;
+        public boolean verbose;
+        public boolean oplogs;
+        public boolean cleanup;
+        public FileType type;
+
+        private Options(String keyspaceName, String cfName)
+        {
+            this.keyspaceName = keyspaceName;
+            this.cfName = cfName;
+        }
+
+        public static Options parseArgs(String cmdArgs[])
+        {
+            CommandLineParser parser = new GnuParser();
+            CmdLineOptions options = getCmdLineOptions();
+            try
+            {
+                CommandLine cmd = parser.parse(options, cmdArgs, false);
+
+                if (cmd.hasOption(HELP_OPTION))
+                {
+                    printUsage(options);
+                    System.exit(0);
+                }
+
+                String[] args = cmd.getArgs();
+                if (args.length != 2)
+                {
+                    String msg = args.length < 2 ? "Missing arguments" : "Too many arguments";
+                    System.err.println(msg);
+                    printUsage(options);
+                    System.exit(1);
+                }
+
+                String keyspaceName = args[0];
+                String cfName = args[1];
+
+                Options opts = new Options(keyspaceName, cfName);
+
+                opts.debug = cmd.hasOption(DEBUG_OPTION);
+                opts.verbose = cmd.hasOption(VERBOSE_OPTION);
+                opts.type = FileType.fromOption(cmd.getOptionValue(TYPE_OPTION));
+                opts.oplogs = cmd.hasOption(OP_LOG_OPTION);
+                opts.cleanup = cmd.hasOption(CLEANUP_OPTION);
+
+                return opts;
+            }
+            catch (ParseException e)
+            {
+                errorMsg(e.getMessage(), options);
+                return null;
+            }
+        }
+
+        private static void errorMsg(String msg, CmdLineOptions options)
+        {
+            System.err.println(msg);
+            printUsage(options);
+            System.exit(1);
+        }
+
+        private static CmdLineOptions getCmdLineOptions()
+        {
+            CmdLineOptions options = new CmdLineOptions();
+            options.addOption("c", CLEANUP_OPTION, "clean-up any outstanding transactions");
+            options.addOption("d", DEBUG_OPTION, "display stack traces");
+            options.addOption("h", HELP_OPTION, "display this help message");
+            options.addOption("o", OP_LOG_OPTION, "include operation logs");
+            options.addOption("t", TYPE_OPTION, true, FileType.descr());
+            options.addOption("v", VERBOSE_OPTION, "verbose output");
+
+            return options;
+        }
+
+        public static void printUsage(CmdLineOptions options)
+        {
+            String usage = String.format("%s [options] <keyspace> <column_family>", TOOL_NAME);
+            StringBuilder header = new StringBuilder();
+            header.append("--\n");
+            header.append("List sstable files for the provided table." );
+            header.append("\n--\n");
+            header.append("Options are:");
+            new HelpFormatter().printHelp(usage, header.toString(), options, "");
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
index 9388d98..f64b8d9 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneScrubber.java
@@ -34,7 +34,7 @@ import org.apache.cassandra.db.Directories;
 import org.apache.cassandra.db.Keyspace;
 import org.apache.cassandra.db.compaction.*;
 import org.apache.cassandra.db.lifecycle.LifecycleTransaction;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.cassandra.io.sstable.*;
 import org.apache.cassandra.utils.JVMStabilityInspector;
@@ -84,7 +84,7 @@ public class StandaloneScrubber
             String snapshotName = "pre-scrub-" + System.currentTimeMillis();
 
             OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
-            Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+            Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
 
             List<SSTableReader> sstables = new ArrayList<>();
 
@@ -145,7 +145,7 @@ public class StandaloneScrubber
             // Check (and repair) manifests
             checkManifest(cfs.getCompactionStrategyManager(), cfs, sstables);
             CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             System.exit(0); // We need that to stop non daemonized threads
         }
         catch (Exception e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
index e53038d..a192491 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneSplitter.java
@@ -23,7 +23,7 @@ import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.cassandra.config.Schema;
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.commons.cli.*;
 
@@ -164,7 +164,7 @@ public class StandaloneSplitter
                 }
             }
             CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             System.exit(0); // We need that to stop non daemonized threads
         }
         catch (Exception e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
index 0851d5b..88e34b7 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneUpgrader.java
@@ -20,7 +20,7 @@ package org.apache.cassandra.tools;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.cassandra.db.lifecycle.TransactionLogs;
+import org.apache.cassandra.db.lifecycle.TransactionLog;
 import org.apache.cassandra.io.sstable.format.SSTableReader;
 import org.apache.commons.cli.*;
 
@@ -63,7 +63,7 @@ public class StandaloneUpgrader
             ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cf);
 
             OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
-            Directories.SSTableLister lister = cfs.directories.sstableLister();
+            Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW);
             if (options.snapshot != null)
                 lister.onlyBackups(true).snapshots(options.snapshot);
             else
@@ -120,7 +120,7 @@ public class StandaloneUpgrader
                 }
             }
             CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
-            TransactionLogs.waitForDeletions();
+            TransactionLog.waitForDeletions();
             System.exit(0);
         }
         catch (Exception e)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
index f71f58d..0b17e39 100644
--- a/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
+++ b/src/java/org/apache/cassandra/tools/StandaloneVerifier.java
@@ -69,7 +69,7 @@ public class StandaloneVerifier
             ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cfName);
 
             OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
-            Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+            Directories.SSTableLister lister = cfs.directories.sstableLister(Directories.OnTxnErr.THROW).skipTemporary(true);
 
             boolean extended = options.extended;
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/src/java/org/apache/cassandra/tools/nodetool/Stop.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/nodetool/Stop.java b/src/java/org/apache/cassandra/tools/nodetool/Stop.java
index ad1fc27..6229e65 100644
--- a/src/java/org/apache/cassandra/tools/nodetool/Stop.java
+++ b/src/java/org/apache/cassandra/tools/nodetool/Stop.java
@@ -36,7 +36,7 @@ public class Stop extends NodeToolCmd
 
     @Option(title = "compactionId",
            name = {"-id", "--compaction-id"},
-           description = "Use -id to stop a compaction by the specified id. Ids can be found in the system.compactions_in_progress table.",
+           description = "Use -id to stop a compaction by the specified id. Ids can be found in the transaction log files whose name starts with compaction_, located in the table transactions folder.",
            required = false)
     private String compactionId = "";
 

http://git-wip-us.apache.org/repos/asf/cassandra/blob/5726625a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
index 47c07ac..9e36bab 100644
--- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
+++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
@@ -532,7 +532,7 @@ public class ColumnFamilyStoreTest
 
         ColumnFamilyStore.scrubDataDirectories(cfs.metadata);
 
-        List<File> ssTableFiles = new Directories(cfs.metadata).sstableLister().listFiles();
+        List<File> ssTableFiles = new Directories(cfs.metadata).sstableLister(Directories.OnTxnErr.THROW).listFiles();
         assertNotNull(ssTableFiles);
         assertEquals(0, ssTableFiles.size());
     }