[jira] [Commented] (CASSANDRA-11120) Leak Detected while bringing nodes up and down when under stress.

["Ariel Weisberg (JIRA)" <ji...@apache.org>]

    [ https://issues.apache.org/jira/browse/CASSANDRA-11120?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15133120#comment-15133120 ] 

Ariel Weisberg commented on CASSANDRA-11120:
--------------------------------------------

The path. Now with more line endings. The format of the output is "<field type> <containing class name>.<field name>-<field value type>". The first value is not in this format. It is a name string defined by the tidier.

It looks like {{SSTableReader$GlobalTidy}} has a path to {{SSTableReader$InstanceTidy}}. It should not have a path to the object being tidied ({{SSTableReader}}). The path is because {{readerMeterSyncFuture}} leads out to places we would rather it didn't. I think can add a WeakReference and update the {{Ref.Visitor}} to ignore weak references. There is some trickiness there because {{com.sun.rmi.transport.WeakRef}} extends {{java.lang.ref.WeakReference}} to allow pinning/unpinning a weak reference by storing an additional strong reference.

{noformat}
[/var/lib/cassandra/data/keyspace1/standard1-7cddd4c1cacc11e5aa69f375b464842d/ma-14-big
 private java.util.concurrent.ScheduledFuture org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy.readMeterSyncFuture-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask
 final java.util.concurrent.ScheduledThreadPoolExecutor java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.this$0-java.util.concurrent.ScheduledThreadPoolExecutor
 private final java.util.HashSet java.util.concurrent.ThreadPoolExecutor.workers-java.util.HashSet
 private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap
 transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;
 transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node
 final java.lang.Object java.util.HashMap$Node.key-java.util.concurrent.ThreadPoolExecutor$Worker
 final java.lang.Thread java.util.concurrent.ThreadPoolExecutor$Worker.thread-java.lang.Thread
 private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup
 private final java.lang.ThreadGroup java.lang.ThreadGroup.parent-java.lang.ThreadGroup
 java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;
 java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread
 private java.lang.Runnable java.lang.Thread.target-java.util.concurrent.ThreadPoolExecutor$Worker
 final java.util.concurrent.ThreadPoolExecutor java.util.concurrent.ThreadPoolExecutor$Worker.this$0-java.util.concurrent.ScheduledThreadPoolExecutor
 private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue
 private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask
 private java.util.concurrent.Callable java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter
 final java.lang.Runnable java.util.concurrent.Executors$RunnableAdapter.task-sun.rmi.transport.DGCImpl$1
 final sun.rmi.transport.DGCImpl sun.rmi.transport.DGCImpl$1.this$0-sun.rmi.transport.DGCImpl
 private java.util.Map sun.rmi.transport.DGCImpl.leaseTable-java.util.HashMap
 transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;
 transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node
 java.lang.Object java.util.HashMap$Node.value-sun.rmi.transport.DGCImpl$LeaseInfo
 java.util.Set sun.rmi.transport.DGCImpl$LeaseInfo.notifySet-java.util.HashSet
 private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap
 transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;
 transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node
 final java.lang.Object java.util.HashMap$Node.key-sun.rmi.transport.Target
 private final sun.rmi.transport.WeakRef sun.rmi.transport.Target.weakImpl-sun.rmi.transport.WeakRef
 private java.lang.Object sun.rmi.transport.WeakRef.strongRef-javax.management.remote.rmi.RMIJRMPServerImpl
 private final java.util.List javax.management.remote.rmi.RMIServerImpl.clientList-java.util.ArrayList
 transient java.lang.Object[] java.util.ArrayList.elementData-[Ljava.lang.Object;
 transient java.lang.Object[] java.util.ArrayList.elementData-java.lang.ref.WeakReference
 private java.lang.Object java.lang.ref.Reference.referent-javax.management.remote.rmi.RMIConnectionImpl
 private final javax.management.MBeanServer javax.management.remote.rmi.RMIConnectionImpl.mbeanServer-com.sun.jmx.mbeanserver.JmxMBeanServer
 private volatile javax.management.MBeanServer com.sun.jmx.mbeanserver.JmxMBeanServer.mbsInterceptor-com.sun.jmx.interceptor.DefaultMBeanServerInterceptor
 private final transient com.sun.jmx.mbeanserver.Repository com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.repository-com.sun.jmx.mbeanserver.Repository
 private final java.util.Map com.sun.jmx.mbeanserver.Repository.domainTb-java.util.HashMap
 transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;
 transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node
 java.lang.Object java.util.HashMap$Node.value-java.util.HashMap
 transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;
 transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node
 java.lang.Object java.util.HashMap$Node.value-com.sun.jmx.mbeanserver.NamedObject
 private final javax.management.DynamicMBean com.sun.jmx.mbeanserver.NamedObject.object-com.sun.jmx.mbeanserver.StandardMBeanSupport
 private final java.lang.Object com.sun.jmx.mbeanserver.MBeanSupport.resource-org.apache.cassandra.db.ColumnFamilyStore
 public final org.apache.cassandra.db.Keyspace org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace
 private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap
 private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore
 private final java.util.concurrent.ScheduledFuture org.apache.cassandra.db.ColumnFamilyStore.latencyCalculator-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask
 final java.util.concurrent.ScheduledThreadPoolExecutor java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.this$0-org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor
 private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue
 private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask
 private java.util.concurrent.Callable java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter
 final java.lang.Runnable java.util.concurrent.Executors$RunnableAdapter.task-org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable
 private final java.lang.Runnable org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable.runnable-org.apache.cassandra.db.ColumnFamilyStore$3
 final org.apache.cassandra.db.ColumnFamilyStore org.apache.cassandra.db.ColumnFamilyStore$3.this$0-org.apache.cassandra.db.ColumnFamilyStore
 public final org.apache.cassandra.db.Keyspace org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace
 private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap
 private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore
 private final org.apache.cassandra.db.lifecycle.Tracker org.apache.cassandra.db.ColumnFamilyStore.data-org.apache.cassandra.db.lifecycle.Tracker
 public final java.util.Collection org.apache.cassandra.db.lifecycle.Tracker.subscribers-java.util.concurrent.CopyOnWriteArrayList
 public final java.util.Collection org.apache.cassandra.db.lifecycle.Tracker.subscribers-org.apache.cassandra.db.compaction.CompactionStrategyManager
 private volatile org.apache.cassandra.db.compaction.AbstractCompactionStrategy org.apache.cassandra.db.compaction.CompactionStrategyManager.repaired-org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy
 private final java.util.Set org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.sstables-java.util.HashSet
 private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap
 transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;
 transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node
 final java.lang.Object java.util.HashMap$Node.key-org.apache.cassandra.io.sstable.format.big.BigTableReader
 private final org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier org.apache.cassandra.io.sstable.format.SSTableReader.tidy-org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier
 private org.apache.cassandra.utils.concurrent.Ref org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier.globalRef-org.apache.cassandra.utils.concurrent.Ref]
{noformat}

> Leak Detected while bringing nodes up and down when under stress.
> -----------------------------------------------------------------
>
>                 Key: CASSANDRA-11120
>                 URL: https://issues.apache.org/jira/browse/CASSANDRA-11120
>             Project: Cassandra
>          Issue Type: Bug
>          Components: Core
>            Reporter: Jeremiah Jordan
>            Priority: Minor
>             Fix For: 3.0.x
>
>
> So after CASSANDRA-10688 has been fixed I'm able to reproduce some leaks consistently with my stress test suite that is doing repairs/stress/bringing nodes up and down.
> One such example:
> {noformat}
> ERROR [Strong-Reference-Leak-Detector:1] 2016-02-03 23:32:38,827  NoSpamLogger.java:97 - Strong self-ref loop detected [/var/lib/cassandra/data/keyspace1/standard1-7cddd4c1cacc11e5aa69f375b464842d/ma-14-big, private java.util.concurrent.ScheduledFuture org.apache.cassandra.io.sstable.format.SSTableReader$GlobalTidy.readMeterSyncFuture-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask, final java.util.concurrent.ScheduledThreadPoolExecutor java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.this$0-java.util.concurrent.ScheduledThreadPoolExecutor, private final java.util.HashSet java.util.concurrent.ThreadPoolExecutor.workers-java.util.HashSet, private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, final java.lang.Object java.util.HashMap$Node.key-java.util.concurrent.ThreadPoolExecutor$Worker, final java.lang.Thread java.util.concurrent.ThreadPoolExecutor$Worker.thread-java.lang.Thread, private java.lang.ThreadGroup java.lang.Thread.group-java.lang.ThreadGroup, private final java.lang.ThreadGroup java.lang.ThreadGroup.parent-java.lang.ThreadGroup, java.lang.Thread[] java.lang.ThreadGroup.threads-[Ljava.lang.Thread;, java.lang.Thread[] java.lang.ThreadGroup.threads-java.lang.Thread, private java.lang.Runnable java.lang.Thread.target-java.util.concurrent.ThreadPoolExecutor$Worker, final java.util.concurrent.ThreadPoolExecutor java.util.concurrent.ThreadPoolExecutor$Worker.this$0-java.util.concurrent.ScheduledThreadPoolExecutor, private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue, private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask, private java.util.concurrent.Callable java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter, final java.lang.Runnable java.util.concurrent.Executors$RunnableAdapter.task-sun.rmi.transport.DGCImpl$1, final sun.rmi.transport.DGCImpl sun.rmi.transport.DGCImpl$1.this$0-sun.rmi.transport.DGCImpl, private java.util.Map sun.rmi.transport.DGCImpl.leaseTable-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, java.lang.Object java.util.HashMap$Node.value-sun.rmi.transport.DGCImpl$LeaseInfo, java.util.Set sun.rmi.transport.DGCImpl$LeaseInfo.notifySet-java.util.HashSet, private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, final java.lang.Object java.util.HashMap$Node.key-sun.rmi.transport.Target, private final sun.rmi.transport.WeakRef sun.rmi.transport.Target.weakImpl-sun.rmi.transport.WeakRef, private java.lang.Object sun.rmi.transport.WeakRef.strongRef-javax.management.remote.rmi.RMIJRMPServerImpl, private final java.util.List javax.management.remote.rmi.RMIServerImpl.clientList-java.util.ArrayList, transient java.lang.Object[] java.util.ArrayList.elementData-[Ljava.lang.Object;, transient java.lang.Object[] java.util.ArrayList.elementData-java.lang.ref.WeakReference, private java.lang.Object java.lang.ref.Reference.referent-javax.management.remote.rmi.RMIConnectionImpl, private final javax.management.MBeanServer javax.management.remote.rmi.RMIConnectionImpl.mbeanServer-com.sun.jmx.mbeanserver.JmxMBeanServer, private volatile javax.management.MBeanServer com.sun.jmx.mbeanserver.JmxMBeanServer.mbsInterceptor-com.sun.jmx.interceptor.DefaultMBeanServerInterceptor, private final transient com.sun.jmx.mbeanserver.Repository com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.repository-com.sun.jmx.mbeanserver.Repository, private final java.util.Map com.sun.jmx.mbeanserver.Repository.domainTb-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, java.lang.Object java.util.HashMap$Node.value-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, java.lang.Object java.util.HashMap$Node.value-com.sun.jmx.mbeanserver.NamedObject, private final javax.management.DynamicMBean com.sun.jmx.mbeanserver.NamedObject.object-com.sun.jmx.mbeanserver.StandardMBeanSupport, private final java.lang.Object com.sun.jmx.mbeanserver.MBeanSupport.resource-org.apache.cassandra.db.ColumnFamilyStore, public final org.apache.cassandra.db.Keyspace org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace, private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap, private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore, private final java.util.concurrent.ScheduledFuture org.apache.cassandra.db.ColumnFamilyStore.latencyCalculator-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask, final java.util.concurrent.ScheduledThreadPoolExecutor java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.this$0-org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor, private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue, private final java.util.concurrent.BlockingQueue java.util.concurrent.ThreadPoolExecutor.workQueue-java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask, private java.util.concurrent.Callable java.util.concurrent.FutureTask.callable-java.util.concurrent.Executors$RunnableAdapter, final java.lang.Runnable java.util.concurrent.Executors$RunnableAdapter.task-org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable, private final java.lang.Runnable org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable.runnable-org.apache.cassandra.db.ColumnFamilyStore$3, final org.apache.cassandra.db.ColumnFamilyStore org.apache.cassandra.db.ColumnFamilyStore$3.this$0-org.apache.cassandra.db.ColumnFamilyStore, public final org.apache.cassandra.db.Keyspace org.apache.cassandra.db.ColumnFamilyStore.keyspace-org.apache.cassandra.db.Keyspace, private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-java.util.concurrent.ConcurrentHashMap, private final java.util.concurrent.ConcurrentMap org.apache.cassandra.db.Keyspace.columnFamilyStores-org.apache.cassandra.db.ColumnFamilyStore, private final org.apache.cassandra.db.lifecycle.Tracker org.apache.cassandra.db.ColumnFamilyStore.data-org.apache.cassandra.db.lifecycle.Tracker, public final java.util.Collection org.apache.cassandra.db.lifecycle.Tracker.subscribers-java.util.concurrent.CopyOnWriteArrayList, public final java.util.Collection org.apache.cassandra.db.lifecycle.Tracker.subscribers-org.apache.cassandra.db.compaction.CompactionStrategyManager, private volatile org.apache.cassandra.db.compaction.AbstractCompactionStrategy org.apache.cassandra.db.compaction.CompactionStrategyManager.repaired-org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy, private final java.util.Set org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy.sstables-java.util.HashSet, private transient java.util.HashMap java.util.HashSet.map-java.util.HashMap, transient java.util.HashMap$Node[] java.util.HashMap.table-[Ljava.util.HashMap$Node;, transient java.util.HashMap$Node[] java.util.HashMap.table-java.util.HashMap$Node, final java.lang.Object java.util.HashMap$Node.key-org.apache.cassandra.io.sstable.format.big.BigTableReader, private final org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier org.apache.cassandra.io.sstable.format.SSTableReader.tidy-org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier, private org.apache.cassandra.utils.concurrent.Ref org.apache.cassandra.io.sstable.format.SSTableReader$InstanceTidier.globalRef-org.apache.cassandra.utils.concurrent.Ref]
> {noformat}
> CC [~benedict]



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)