You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@sirona.apache.org by Romain Manni-Bucau <rm...@gmail.com> on 2014/03/07 08:18:46 UTC
Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Hi
any issue making disruptor optional and creating two
CubePathTrackingDataStore impl (the old one and disruptor one)? Think
we shouldn't make disruptor mandatory excepted if perf are > 10.
Romain Manni-Bucau
Twitter: @rmannibucau
Blog: http://rmannibucau.wordpress.com/
LinkedIn: http://fr.linkedin.com/in/rmannibucau
Github: https://github.com/rmannibucau
---------- Forwarded message ----------
From: <ol...@apache.org>
Date: 2014-03-07 7:07 GMT+01:00
Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/
agent/store/cube/src/main/java/org/apache/sirona/cube/
core/src/main/java/org/apache/sirona/tracking/
To: commits@sirona.incubator.apache.org
Author: olamy
Date: Fri Mar 7 06:07:56 2014
New Revision: 1575186
URL: http://svn.apache.org/r1575186
Log:
use disruptor to handle amount of messages and not block the
application when sending path tracking entries
Modified:
incubator/sirona/trunk/agent/store/cube/pom.xml
incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
URL: http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
==============================================================================
--- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
+++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7 06:07:56 2014
@@ -32,6 +32,13 @@
<dependency>
<groupId>org.apache.sirona</groupId>
<artifactId>sirona-core</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.lmax</groupId>
+ <artifactId>disruptor</artifactId>
+ <version>3.2.0</version>
</dependency>
<dependency>
@@ -69,6 +76,34 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createSourcesJar>true</createSourcesJar>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>shaded</shadedClassifierName>
+ <relocations>
+ <relocation>
+ <pattern>com.lmax</pattern>
+ <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
+ </relocation>
+ <!--
+ <relocation>
+ <pattern>org.apache.sirona</pattern>
+ <shadedPattern>org.apache.sirona.sirona</shadedPattern>
+ </relocation>
+ -->
+ </relocations>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
<pluginManagement>
<plugins>
Modified: incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
URL: http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
==============================================================================
--- incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
(original)
+++ incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
Fri Mar 7 06:07:56 2014
@@ -17,6 +17,13 @@
package org.apache.sirona.cube;
+import com.lmax.disruptor.BusySpinWaitStrategy;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.EventHandler;
+import com.lmax.disruptor.EventTranslator;
+import com.lmax.disruptor.RingBuffer;
+import com.lmax.disruptor.dsl.Disruptor;
+import com.lmax.disruptor.dsl.ProducerType;
import org.apache.sirona.configuration.Configuration;
import org.apache.sirona.configuration.ioc.Destroying;
import org.apache.sirona.configuration.ioc.IoCs;
@@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
extends BatchPathTrackingDataStore
implements CollectorPathTrackingDataStore
{
- private final Cube cube = IoCs.findOrCreateInstance(
CubeBuilder.class ).build();
+ private static final Cube CUBE = IoCs.findOrCreateInstance(
CubeBuilder.class ).build();
private static final boolean USE_EXECUTORS = Boolean.parseBoolean(
@@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
Configuration.getProperty(
Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
"false" ) );
+ private static final boolean USE_DISRUPTOR = Boolean.parseBoolean(
+ Configuration.getProperty(
Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.usedisruptor",
"true" ) );
+
protected static ExecutorService executorService;
+ private static RingBuffer<PathTrackingEntry> RINGBUFFER;
+
static
{
@@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
executorService = Executors.newFixedThreadPool( threadsNumber );
}
+
+ if ( USE_DISRUPTOR )
+ {
+ ExecutorService exec = Executors.newCachedThreadPool();
+
+ // FIXME make configurable: ring buffer size and WaitStrategy
+
+ Disruptor<PathTrackingEntry> disruptor =
+ new Disruptor<PathTrackingEntry>( new
EventFactory<PathTrackingEntry>()
+ {
+ @Override
+ public PathTrackingEntry newInstance()
+ {
+ return new PathTrackingEntry();
+ }
+ }, 2048, exec, ProducerType.SINGLE, new BusySpinWaitStrategy()
+ );
+
+ final EventHandler<PathTrackingEntry> handler = new
EventHandler<PathTrackingEntry>()
+ {
+ // event will eventually be recycled by the Disruptor
after it wraps
+ public void onEvent( final PathTrackingEntry entry,
final long sequence, final boolean endOfBatch )
+ throws Exception
+ {
+ CUBE.postBytes( SerializeUtils.serialize( entry
), PathTrackingEntry.class.getName() );
+ }
+ };
+
+ disruptor.handleEventsWith( handler );
+
+ RINGBUFFER = disruptor.start();
+ }
}
@Override
- public void store( PathTrackingEntry pathTrackingEntry )
+ public void store( final PathTrackingEntry pathTrackingEntry )
{
- cube.postBytes( SerializeUtils.serialize( pathTrackingEntry
), PathTrackingEntry.class.getName() );
+ if ( USE_DISRUPTOR )
+ {
+
+ RINGBUFFER.publishEvent( new EventTranslator<PathTrackingEntry>()
+ {
+ @Override
+ public void translateTo( PathTrackingEntry event,
long sequence )
+ {
+ event.setClassName( pathTrackingEntry.getClassName() );
+ event.setExecutionTime(
pathTrackingEntry.getExecutionTime() );
+ event.setLevel( pathTrackingEntry.getLevel() );
+ event.setMethodName( pathTrackingEntry.getMethodName() );
+ event.setNodeId( pathTrackingEntry.getNodeId() );
+ event.setStartTime( pathTrackingEntry.getStartTime() );
+ event.setTrackingId( pathTrackingEntry.getTrackingId() );
+ }
+ } );
+
+
+ }
+ else
+ {
+ CUBE.postBytes( SerializeUtils.serialize(
pathTrackingEntry ), PathTrackingEntry.class.getName() );
+ }
+
}
@Override
protected void pushEntriesByBatch( Map<String, List<Pointer>>
pathTrackingEntries )
{
- if (!USE_SINGLE_STORE)
+ if ( !USE_SINGLE_STORE )
{
for ( Map.Entry<String, List<Pointer>> entry :
pathTrackingEntries.entrySet() )
@@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
{
if ( !pointer.isFree() )
{
- cube.postBytes( readBytes( pointer ),
PathTrackingEntry.class.getName() );
+ CUBE.postBytes( readBytes( pointer ),
PathTrackingEntry.class.getName() );
pointer.freeMemory();
}
}
@@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
{
executorService.shutdownNow();
}
+
+
}
Modified: incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
URL: http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
==============================================================================
--- incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
(original)
+++ incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
Fri Mar 7 06:07:56 2014
@@ -59,7 +59,6 @@ public class PathTracker
private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
Configuration.getProperty(
Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
"false" ) );
-
protected static ExecutorService executorService;
static
Modified: incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
URL: http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
==============================================================================
--- incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
(original)
+++ incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
Fri Mar 7 06:07:56 2014
@@ -63,6 +63,11 @@ public class PathTrackingEntry
*/
private int level;
+ public PathTrackingEntry()
+ {
+ // no op
+ }
+
public PathTrackingEntry( String trackingId, String nodeId,
String className, String methodName, long startTime,
long executionTime, int level )
{
Re: Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Posted by Romain Manni-Bucau <rm...@gmail.com>.
Yep but if you allocate a heap of 1G and the app uses 10G off heap you have
a biggest issue. A fixed size buffer would be better, even if loosing some
messages, no?
Le 11 mars 2014 12:45, "Olivier Lamy" <ol...@apache.org> a écrit :
> On 11 March 2014 22:33, Romain Manni-Bucau <rm...@gmail.com> wrote:
>
> > great
> >
> > btw i don't get the store is not a solution, store doesn't mean
> > synchronous or so. We have a lot of async store actually.
> >
> > I know you think I insist but "I reduce the memory": do you have
> > figures here too? Using Unsafe is quite impacting (since then you need
> > more mem than the one you define on the JVM etc so it makes config
> > harder to do) so I'd love to be able to justify it by numbers.
>
>
> How a byte array resulting from the serialisation of an object having only
> String fields (and store outside of the heap) can consume more memory than
> tons of objects stored in the heap ? :-)
> I just believe such agent must have the minimum memory impact on the real
> application if not they won't be used.
>
>
>
>
> > Romain Manni-Bucau
> > Twitter: @rmannibucau
> > Blog: http://rmannibucau.wordpress.com/
> > LinkedIn: http://fr.linkedin.com/in/rmannibucau
> > Github: https://github.com/rmannibucau
> >
> >
> >
> > 2014-03-11 12:26 GMT+01:00 Olivier Lamy <ol...@apache.org>:
> > > On 11 March 2014 17:07, Romain Manni-Bucau <rm...@gmail.com>
> > wrote:
> > >> I dont doubt about boot but what I saw was that then ringbuffer can
> slow
> > >> down or loose message. Normally you configure path tracking to capture
> > only
> > > I know but I don't want to filter (track everything :-) )
> > >> what you want and then the storage is done asynchronously so perf
> > qhouldnt
> > >> be that bad. If it is we should just do more async (using the store).
> > Well
> > >> get the drawbacks of queue but these are known and controllable.
> > >
> > > That's why the store is just not an acceptable solution. I reduce the
> > > memory by storing only bytes (serialization of the objet and send it
> > > as it to the collector).
> > > My goal is to have a sort of "streaming processing" of those entries
> > > (ideally no more local storage).
> > >
> > > Anyway I made disruptor optional and not default.
> > >
> > >>
> > >> Wdyt?
> > >> Le 10 mars 2014 23:16, "Olivier Lamy" <ol...@apache.org> a écrit :
> > >>
> > >>> Hi
> > >>> Apologize for delayed response. I was a bit off during the weekend.
> > >>>
> > >>> On 7 March 2014 20:17, Romain Manni-Bucau <rm...@gmail.com>
> > wrote:
> > >>> > well if you have figures and say me disruptor is always 10 times
> > >>> > faster than default impl (regarding overall jvm perf and not only
> > this
> > >>> > thread) I'll say: ok keep disruptor only but since i'm not sure it
> > >>> > changes a lot of things i think default shouldnt use disruptor and
> we
> > >>> > should stay simple and don't bring disruptor transitively
> > >>>
> > >>> I don't have any figures except starting a tomcat locally with path
> > >>> tracking activated now works with disruptor :-)
> > >>> I can make it optional it's a big problem but that's really more
> > >>> performant (again not mathematic figures just based on the fact it
> > >>> works better locally :-) )
> > >>>
> > >>>
> > >>> > Romain Manni-Bucau
> > >>> > Twitter: @rmannibucau
> > >>> > Blog: http://rmannibucau.wordpress.com/
> > >>> > LinkedIn: http://fr.linkedin.com/in/rmannibucau
> > >>> > Github: https://github.com/rmannibucau
> > >>> >
> > >>> >
> > >>> >
> > >>> > 2014-03-07 9:15 GMT+01:00 Jean-Louis MONTEIRO <jeanouii@gmail.com
> >:
> > >>> >> Romain not sure I got the mandatory with 10.
> > >>> >> BTW big big thx you. Great feature.
> > >>> >> Gonna try to use that and get back to the project.
> > >>> >>
> > >>> >> Jean Louis
> > >>> >> Le 7 mars 2014 08:19, "Romain Manni-Bucau" <rmannibucau@gmail.com
> >
> > a
> > >>> écrit :
> > >>> >>
> > >>> >>> Hi
> > >>> >>>
> > >>> >>> any issue making disruptor optional and creating two
> > >>> >>> CubePathTrackingDataStore impl (the old one and disruptor one)?
> > Think
> > >>> >>> we shouldn't make disruptor mandatory excepted if perf are > 10.
> > >>> >>>
> > >>> >>> Romain Manni-Bucau
> > >>> >>> Twitter: @rmannibucau
> > >>> >>> Blog: http://rmannibucau.wordpress.com/
> > >>> >>> LinkedIn: http://fr.linkedin.com/in/rmannibucau
> > >>> >>> Github: https://github.com/rmannibucau
> > >>> >>>
> > >>> >>>
> > >>> >>>
> > >>> >>>
> > >>> >>> ---------- Forwarded message ----------
> > >>> >>> From: <ol...@apache.org>
> > >>> >>> Date: 2014-03-07 7:07 GMT+01:00
> > >>> >>> Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
> > >>> >>> agent/store/cube/
> > >>> >>> agent/store/cube/src/main/java/org/apache/sirona/cube/
> > >>> >>> core/src/main/java/org/apache/sirona/tracking/
> > >>> >>> To: commits@sirona.incubator.apache.org
> > >>> >>>
> > >>> >>>
> > >>> >>> Author: olamy
> > >>> >>> Date: Fri Mar 7 06:07:56 2014
> > >>> >>> New Revision: 1575186
> > >>> >>>
> > >>> >>> URL: http://svn.apache.org/r1575186
> > >>> >>> Log:
> > >>> >>> use disruptor to handle amount of messages and not block the
> > >>> >>> application when sending path tracking entries
> > >>> >>>
> > >>> >>> Modified:
> > >>> >>> incubator/sirona/trunk/agent/store/cube/pom.xml
> > >>> >>>
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> > >>> >>>
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> > >>> >>>
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> > >>> >>>
> > >>> >>> Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
> > >>> >>> URL:
> > >>> >>>
> > >>>
> >
> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
> > >>> >>>
> > >>> >>>
> > >>>
> >
> ==============================================================================
> > >>> >>> --- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
> > >>> >>> +++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7
> > 06:07:56
> > >>> >>> 2014
> > >>> >>> @@ -32,6 +32,13 @@
> > >>> >>> <dependency>
> > >>> >>> <groupId>org.apache.sirona</groupId>
> > >>> >>> <artifactId>sirona-core</artifactId>
> > >>> >>> + <scope>provided</scope>
> > >>> >>> + </dependency>
> > >>> >>> +
> > >>> >>> + <dependency>
> > >>> >>> + <groupId>com.lmax</groupId>
> > >>> >>> + <artifactId>disruptor</artifactId>
> > >>> >>> + <version>3.2.0</version>
> > >>> >>> </dependency>
> > >>> >>>
> > >>> >>> <dependency>
> > >>> >>> @@ -69,6 +76,34 @@
> > >>> >>> </execution>
> > >>> >>> </executions>
> > >>> >>> </plugin>
> > >>> >>> + <plugin>
> > >>> >>> + <groupId>org.apache.maven.plugins</groupId>
> > >>> >>> + <artifactId>maven-shade-plugin</artifactId>
> > >>> >>> + <executions>
> > >>> >>> + <execution>
> > >>> >>> + <goals>
> > >>> >>> + <goal>shade</goal>
> > >>> >>> + </goals>
> > >>> >>> + <configuration>
> > >>> >>> + <createSourcesJar>true</createSourcesJar>
> > >>> >>> +
> > <shadedArtifactAttached>true</shadedArtifactAttached>
> > >>> >>> +
> <shadedClassifierName>shaded</shadedClassifierName>
> > >>> >>> + <relocations>
> > >>> >>> + <relocation>
> > >>> >>> + <pattern>com.lmax</pattern>
> > >>> >>> +
> > >>> >>> <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
> > >>> >>> + </relocation>
> > >>> >>> + <!--
> > >>> >>> + <relocation>
> > >>> >>> + <pattern>org.apache.sirona</pattern>
> > >>> >>> +
> > >>> <shadedPattern>org.apache.sirona.sirona</shadedPattern>
> > >>> >>> + </relocation>
> > >>> >>> + -->
> > >>> >>> + </relocations>
> > >>> >>> + </configuration>
> > >>> >>> + </execution>
> > >>> >>> + </executions>
> > >>> >>> + </plugin>
> > >>> >>> </plugins>
> > >>> >>> <pluginManagement>
> > >>> >>> <plugins>
> > >>> >>>
> > >>> >>> Modified:
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> > >>> >>> URL:
> > >>> >>>
> > >>>
> >
> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> > >>> >>>
> > >>> >>>
> > >>>
> >
> ==============================================================================
> > >>> >>> ---
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> > >>> >>> (original)
> > >>> >>> +++
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> > >>> >>> Fri Mar 7 06:07:56 2014
> > >>> >>> @@ -17,6 +17,13 @@
> > >>> >>>
> > >>> >>> package org.apache.sirona.cube;
> > >>> >>>
> > >>> >>> +import com.lmax.disruptor.BusySpinWaitStrategy;
> > >>> >>> +import com.lmax.disruptor.EventFactory;
> > >>> >>> +import com.lmax.disruptor.EventHandler;
> > >>> >>> +import com.lmax.disruptor.EventTranslator;
> > >>> >>> +import com.lmax.disruptor.RingBuffer;
> > >>> >>> +import com.lmax.disruptor.dsl.Disruptor;
> > >>> >>> +import com.lmax.disruptor.dsl.ProducerType;
> > >>> >>> import org.apache.sirona.configuration.Configuration;
> > >>> >>> import org.apache.sirona.configuration.ioc.Destroying;
> > >>> >>> import org.apache.sirona.configuration.ioc.IoCs;
> > >>> >>> @@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
> > >>> >>> extends BatchPathTrackingDataStore
> > >>> >>> implements CollectorPathTrackingDataStore
> > >>> >>> {
> > >>> >>> - private final Cube cube = IoCs.findOrCreateInstance(
> > >>> >>> CubeBuilder.class ).build();
> > >>> >>> + private static final Cube CUBE = IoCs.findOrCreateInstance(
> > >>> >>> CubeBuilder.class ).build();
> > >>> >>>
> > >>> >>>
> > >>> >>> private static final boolean USE_EXECUTORS =
> > Boolean.parseBoolean(
> > >>> >>> @@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
> > >>> >>> private static boolean USE_SINGLE_STORE =
> > Boolean.parseBoolean(
> > >>> >>> Configuration.getProperty(
> > >>> >>> Configuration.CONFIG_PROPERTY_PREFIX +
> "pathtracking.singlestore",
> > >>> >>> "false" ) );
> > >>> >>>
> > >>> >>> + private static final boolean USE_DISRUPTOR =
> > Boolean.parseBoolean(
> > >>> >>> + Configuration.getProperty(
> > >>> >>> Configuration.CONFIG_PROPERTY_PREFIX +
> "pathtracking.usedisruptor",
> > >>> >>> "true" ) );
> > >>> >>> +
> > >>> >>> protected static ExecutorService executorService;
> > >>> >>>
> > >>> >>> + private static RingBuffer<PathTrackingEntry> RINGBUFFER;
> > >>> >>> +
> > >>> >>> static
> > >>> >>> {
> > >>> >>>
> > >>> >>> @@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
> > >>> >>> executorService = Executors.newFixedThreadPool(
> > >>> threadsNumber
> > >>> >>> );
> > >>> >>>
> > >>> >>> }
> > >>> >>> +
> > >>> >>> + if ( USE_DISRUPTOR )
> > >>> >>> + {
> > >>> >>> + ExecutorService exec =
> > Executors.newCachedThreadPool();
> > >>> >>> +
> > >>> >>> + // FIXME make configurable: ring buffer size and
> > >>> WaitStrategy
> > >>> >>> +
> > >>> >>> + Disruptor<PathTrackingEntry> disruptor =
> > >>> >>> + new Disruptor<PathTrackingEntry>( new
> > >>> >>> EventFactory<PathTrackingEntry>()
> > >>> >>> + {
> > >>> >>> + @Override
> > >>> >>> + public PathTrackingEntry newInstance()
> > >>> >>> + {
> > >>> >>> + return new PathTrackingEntry();
> > >>> >>> + }
> > >>> >>> + }, 2048, exec, ProducerType.SINGLE, new
> > >>> >>> BusySpinWaitStrategy()
> > >>> >>> + );
> > >>> >>> +
> > >>> >>> + final EventHandler<PathTrackingEntry> handler = new
> > >>> >>> EventHandler<PathTrackingEntry>()
> > >>> >>> + {
> > >>> >>> + // event will eventually be recycled by the
> > Disruptor
> > >>> >>> after it wraps
> > >>> >>> + public void onEvent( final PathTrackingEntry
> > entry,
> > >>> >>> final long sequence, final boolean endOfBatch )
> > >>> >>> + throws Exception
> > >>> >>> + {
> > >>> >>> + CUBE.postBytes( SerializeUtils.serialize(
> > entry
> > >>> >>> ), PathTrackingEntry.class.getName() );
> > >>> >>> + }
> > >>> >>> + };
> > >>> >>> +
> > >>> >>> + disruptor.handleEventsWith( handler );
> > >>> >>> +
> > >>> >>> + RINGBUFFER = disruptor.start();
> > >>> >>> + }
> > >>> >>> }
> > >>> >>>
> > >>> >>> @Override
> > >>> >>> - public void store( PathTrackingEntry pathTrackingEntry )
> > >>> >>> + public void store( final PathTrackingEntry
> pathTrackingEntry )
> > >>> >>> {
> > >>> >>> - cube.postBytes( SerializeUtils.serialize(
> > pathTrackingEntry
> > >>> >>> ), PathTrackingEntry.class.getName() );
> > >>> >>> + if ( USE_DISRUPTOR )
> > >>> >>> + {
> > >>> >>> +
> > >>> >>> + RINGBUFFER.publishEvent( new
> > >>> >>> EventTranslator<PathTrackingEntry>()
> > >>> >>> + {
> > >>> >>> + @Override
> > >>> >>> + public void translateTo( PathTrackingEntry
> event,
> > >>> >>> long sequence )
> > >>> >>> + {
> > >>> >>> + event.setClassName(
> > >>> pathTrackingEntry.getClassName()
> > >>> >>> );
> > >>> >>> + event.setExecutionTime(
> > >>> >>> pathTrackingEntry.getExecutionTime() );
> > >>> >>> + event.setLevel( pathTrackingEntry.getLevel()
> > );
> > >>> >>> + event.setMethodName(
> > >>> >>> pathTrackingEntry.getMethodName() );
> > >>> >>> + event.setNodeId(
> > pathTrackingEntry.getNodeId() );
> > >>> >>> + event.setStartTime(
> > >>> pathTrackingEntry.getStartTime()
> > >>> >>> );
> > >>> >>> + event.setTrackingId(
> > >>> >>> pathTrackingEntry.getTrackingId() );
> > >>> >>> + }
> > >>> >>> + } );
> > >>> >>> +
> > >>> >>> +
> > >>> >>> + }
> > >>> >>> + else
> > >>> >>> + {
> > >>> >>> + CUBE.postBytes( SerializeUtils.serialize(
> > >>> >>> pathTrackingEntry ), PathTrackingEntry.class.getName() );
> > >>> >>> + }
> > >>> >>> +
> > >>> >>> }
> > >>> >>>
> > >>> >>> @Override
> > >>> >>> protected void pushEntriesByBatch( Map<String,
> List<Pointer>>
> > >>> >>> pathTrackingEntries )
> > >>> >>> {
> > >>> >>> - if (!USE_SINGLE_STORE)
> > >>> >>> + if ( !USE_SINGLE_STORE )
> > >>> >>> {
> > >>> >>>
> > >>> >>> for ( Map.Entry<String, List<Pointer>> entry :
> > >>> >>> pathTrackingEntries.entrySet() )
> > >>> >>> @@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
> > >>> >>> {
> > >>> >>> if ( !pointer.isFree() )
> > >>> >>> {
> > >>> >>> - cube.postBytes( readBytes( pointer ),
> > >>> >>> PathTrackingEntry.class.getName() );
> > >>> >>> + CUBE.postBytes( readBytes( pointer ),
> > >>> >>> PathTrackingEntry.class.getName() );
> > >>> >>> pointer.freeMemory();
> > >>> >>> }
> > >>> >>> }
> > >>> >>> @@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
> > >>> >>> {
> > >>> >>> executorService.shutdownNow();
> > >>> >>> }
> > >>> >>> +
> > >>> >>> +
> > >>> >>> }
> > >>> >>>
> > >>> >>> Modified:
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> > >>> >>> URL:
> > >>> >>>
> > >>>
> >
> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> > >>> >>>
> > >>> >>>
> > >>>
> >
> ==============================================================================
> > >>> >>> ---
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> > >>> >>> (original)
> > >>> >>> +++
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> > >>> >>> Fri Mar 7 06:07:56 2014
> > >>> >>> @@ -59,7 +59,6 @@ public class PathTracker
> > >>> >>> private static boolean USE_SINGLE_STORE =
> > Boolean.parseBoolean(
> > >>> >>> Configuration.getProperty(
> > >>> >>> Configuration.CONFIG_PROPERTY_PREFIX +
> "pathtracking.singlestore",
> > >>> >>> "false" ) );
> > >>> >>>
> > >>> >>> -
> > >>> >>> protected static ExecutorService executorService;
> > >>> >>>
> > >>> >>> static
> > >>> >>>
> > >>> >>> Modified:
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> > >>> >>> URL:
> > >>> >>>
> > >>>
> >
> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> > >>> >>>
> > >>> >>>
> > >>>
> >
> ==============================================================================
> > >>> >>> ---
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> > >>> >>> (original)
> > >>> >>> +++
> > >>> >>>
> > >>>
> >
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> > >>> >>> Fri Mar 7 06:07:56 2014
> > >>> >>> @@ -63,6 +63,11 @@ public class PathTrackingEntry
> > >>> >>> */
> > >>> >>> private int level;
> > >>> >>>
> > >>> >>> + public PathTrackingEntry()
> > >>> >>> + {
> > >>> >>> + // no op
> > >>> >>> + }
> > >>> >>> +
> > >>> >>> public PathTrackingEntry( String trackingId, String nodeId,
> > >>> >>> String className, String methodName, long startTime,
> > >>> >>> long executionTime, int level )
> > >>> >>> {
> > >>> >>>
> > >>>
> > >>>
> > >>>
> > >>> --
> > >>> Olivier Lamy
> > >>> Ecetera: http://ecetera.com.au
> > >>> http://twitter.com/olamy | http://linkedin.com/in/olamy
> > >>>
> > >
> > >
> > >
> > > --
> > > Olivier Lamy
> > > Ecetera: http://ecetera.com.au
> > > http://twitter.com/olamy | http://linkedin.com/in/olamy
> >
>
>
>
> --
> Olivier Lamy
> Ecetera: http://ecetera.com.au
> http://twitter.com/olamy | http://linkedin.com/in/olamy
>
Re: Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Posted by Olivier Lamy <ol...@apache.org>.
On 11 March 2014 22:33, Romain Manni-Bucau <rm...@gmail.com> wrote:
> great
>
> btw i don't get the store is not a solution, store doesn't mean
> synchronous or so. We have a lot of async store actually.
>
> I know you think I insist but "I reduce the memory": do you have
> figures here too? Using Unsafe is quite impacting (since then you need
> more mem than the one you define on the JVM etc so it makes config
> harder to do) so I'd love to be able to justify it by numbers.
How a byte array resulting from the serialisation of an object having only
String fields (and store outside of the heap) can consume more memory than
tons of objects stored in the heap ? :-)
I just believe such agent must have the minimum memory impact on the real
application if not they won't be used.
> Romain Manni-Bucau
> Twitter: @rmannibucau
> Blog: http://rmannibucau.wordpress.com/
> LinkedIn: http://fr.linkedin.com/in/rmannibucau
> Github: https://github.com/rmannibucau
>
>
>
> 2014-03-11 12:26 GMT+01:00 Olivier Lamy <ol...@apache.org>:
> > On 11 March 2014 17:07, Romain Manni-Bucau <rm...@gmail.com>
> wrote:
> >> I dont doubt about boot but what I saw was that then ringbuffer can slow
> >> down or loose message. Normally you configure path tracking to capture
> only
> > I know but I don't want to filter (track everything :-) )
> >> what you want and then the storage is done asynchronously so perf
> qhouldnt
> >> be that bad. If it is we should just do more async (using the store).
> Well
> >> get the drawbacks of queue but these are known and controllable.
> >
> > That's why the store is just not an acceptable solution. I reduce the
> > memory by storing only bytes (serialization of the objet and send it
> > as it to the collector).
> > My goal is to have a sort of "streaming processing" of those entries
> > (ideally no more local storage).
> >
> > Anyway I made disruptor optional and not default.
> >
> >>
> >> Wdyt?
> >> Le 10 mars 2014 23:16, "Olivier Lamy" <ol...@apache.org> a écrit :
> >>
> >>> Hi
> >>> Apologize for delayed response. I was a bit off during the weekend.
> >>>
> >>> On 7 March 2014 20:17, Romain Manni-Bucau <rm...@gmail.com>
> wrote:
> >>> > well if you have figures and say me disruptor is always 10 times
> >>> > faster than default impl (regarding overall jvm perf and not only
> this
> >>> > thread) I'll say: ok keep disruptor only but since i'm not sure it
> >>> > changes a lot of things i think default shouldnt use disruptor and we
> >>> > should stay simple and don't bring disruptor transitively
> >>>
> >>> I don't have any figures except starting a tomcat locally with path
> >>> tracking activated now works with disruptor :-)
> >>> I can make it optional it's a big problem but that's really more
> >>> performant (again not mathematic figures just based on the fact it
> >>> works better locally :-) )
> >>>
> >>>
> >>> > Romain Manni-Bucau
> >>> > Twitter: @rmannibucau
> >>> > Blog: http://rmannibucau.wordpress.com/
> >>> > LinkedIn: http://fr.linkedin.com/in/rmannibucau
> >>> > Github: https://github.com/rmannibucau
> >>> >
> >>> >
> >>> >
> >>> > 2014-03-07 9:15 GMT+01:00 Jean-Louis MONTEIRO <je...@gmail.com>:
> >>> >> Romain not sure I got the mandatory with 10.
> >>> >> BTW big big thx you. Great feature.
> >>> >> Gonna try to use that and get back to the project.
> >>> >>
> >>> >> Jean Louis
> >>> >> Le 7 mars 2014 08:19, "Romain Manni-Bucau" <rm...@gmail.com>
> a
> >>> écrit :
> >>> >>
> >>> >>> Hi
> >>> >>>
> >>> >>> any issue making disruptor optional and creating two
> >>> >>> CubePathTrackingDataStore impl (the old one and disruptor one)?
> Think
> >>> >>> we shouldn't make disruptor mandatory excepted if perf are > 10.
> >>> >>>
> >>> >>> Romain Manni-Bucau
> >>> >>> Twitter: @rmannibucau
> >>> >>> Blog: http://rmannibucau.wordpress.com/
> >>> >>> LinkedIn: http://fr.linkedin.com/in/rmannibucau
> >>> >>> Github: https://github.com/rmannibucau
> >>> >>>
> >>> >>>
> >>> >>>
> >>> >>>
> >>> >>> ---------- Forwarded message ----------
> >>> >>> From: <ol...@apache.org>
> >>> >>> Date: 2014-03-07 7:07 GMT+01:00
> >>> >>> Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
> >>> >>> agent/store/cube/
> >>> >>> agent/store/cube/src/main/java/org/apache/sirona/cube/
> >>> >>> core/src/main/java/org/apache/sirona/tracking/
> >>> >>> To: commits@sirona.incubator.apache.org
> >>> >>>
> >>> >>>
> >>> >>> Author: olamy
> >>> >>> Date: Fri Mar 7 06:07:56 2014
> >>> >>> New Revision: 1575186
> >>> >>>
> >>> >>> URL: http://svn.apache.org/r1575186
> >>> >>> Log:
> >>> >>> use disruptor to handle amount of messages and not block the
> >>> >>> application when sending path tracking entries
> >>> >>>
> >>> >>> Modified:
> >>> >>> incubator/sirona/trunk/agent/store/cube/pom.xml
> >>> >>>
> >>> >>>
> >>>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> >>> >>>
> >>> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> >>> >>>
> >>> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> >>> >>>
> >>> >>> Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
> >>> >>> URL:
> >>> >>>
> >>>
> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
> >>> >>>
> >>> >>>
> >>>
> ==============================================================================
> >>> >>> --- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
> >>> >>> +++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7
> 06:07:56
> >>> >>> 2014
> >>> >>> @@ -32,6 +32,13 @@
> >>> >>> <dependency>
> >>> >>> <groupId>org.apache.sirona</groupId>
> >>> >>> <artifactId>sirona-core</artifactId>
> >>> >>> + <scope>provided</scope>
> >>> >>> + </dependency>
> >>> >>> +
> >>> >>> + <dependency>
> >>> >>> + <groupId>com.lmax</groupId>
> >>> >>> + <artifactId>disruptor</artifactId>
> >>> >>> + <version>3.2.0</version>
> >>> >>> </dependency>
> >>> >>>
> >>> >>> <dependency>
> >>> >>> @@ -69,6 +76,34 @@
> >>> >>> </execution>
> >>> >>> </executions>
> >>> >>> </plugin>
> >>> >>> + <plugin>
> >>> >>> + <groupId>org.apache.maven.plugins</groupId>
> >>> >>> + <artifactId>maven-shade-plugin</artifactId>
> >>> >>> + <executions>
> >>> >>> + <execution>
> >>> >>> + <goals>
> >>> >>> + <goal>shade</goal>
> >>> >>> + </goals>
> >>> >>> + <configuration>
> >>> >>> + <createSourcesJar>true</createSourcesJar>
> >>> >>> +
> <shadedArtifactAttached>true</shadedArtifactAttached>
> >>> >>> + <shadedClassifierName>shaded</shadedClassifierName>
> >>> >>> + <relocations>
> >>> >>> + <relocation>
> >>> >>> + <pattern>com.lmax</pattern>
> >>> >>> +
> >>> >>> <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
> >>> >>> + </relocation>
> >>> >>> + <!--
> >>> >>> + <relocation>
> >>> >>> + <pattern>org.apache.sirona</pattern>
> >>> >>> +
> >>> <shadedPattern>org.apache.sirona.sirona</shadedPattern>
> >>> >>> + </relocation>
> >>> >>> + -->
> >>> >>> + </relocations>
> >>> >>> + </configuration>
> >>> >>> + </execution>
> >>> >>> + </executions>
> >>> >>> + </plugin>
> >>> >>> </plugins>
> >>> >>> <pluginManagement>
> >>> >>> <plugins>
> >>> >>>
> >>> >>> Modified:
> >>> >>>
> >>>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> >>> >>> URL:
> >>> >>>
> >>>
> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> >>> >>>
> >>> >>>
> >>>
> ==============================================================================
> >>> >>> ---
> >>> >>>
> >>>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> >>> >>> (original)
> >>> >>> +++
> >>> >>>
> >>>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> >>> >>> Fri Mar 7 06:07:56 2014
> >>> >>> @@ -17,6 +17,13 @@
> >>> >>>
> >>> >>> package org.apache.sirona.cube;
> >>> >>>
> >>> >>> +import com.lmax.disruptor.BusySpinWaitStrategy;
> >>> >>> +import com.lmax.disruptor.EventFactory;
> >>> >>> +import com.lmax.disruptor.EventHandler;
> >>> >>> +import com.lmax.disruptor.EventTranslator;
> >>> >>> +import com.lmax.disruptor.RingBuffer;
> >>> >>> +import com.lmax.disruptor.dsl.Disruptor;
> >>> >>> +import com.lmax.disruptor.dsl.ProducerType;
> >>> >>> import org.apache.sirona.configuration.Configuration;
> >>> >>> import org.apache.sirona.configuration.ioc.Destroying;
> >>> >>> import org.apache.sirona.configuration.ioc.IoCs;
> >>> >>> @@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
> >>> >>> extends BatchPathTrackingDataStore
> >>> >>> implements CollectorPathTrackingDataStore
> >>> >>> {
> >>> >>> - private final Cube cube = IoCs.findOrCreateInstance(
> >>> >>> CubeBuilder.class ).build();
> >>> >>> + private static final Cube CUBE = IoCs.findOrCreateInstance(
> >>> >>> CubeBuilder.class ).build();
> >>> >>>
> >>> >>>
> >>> >>> private static final boolean USE_EXECUTORS =
> Boolean.parseBoolean(
> >>> >>> @@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
> >>> >>> private static boolean USE_SINGLE_STORE =
> Boolean.parseBoolean(
> >>> >>> Configuration.getProperty(
> >>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
> >>> >>> "false" ) );
> >>> >>>
> >>> >>> + private static final boolean USE_DISRUPTOR =
> Boolean.parseBoolean(
> >>> >>> + Configuration.getProperty(
> >>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.usedisruptor",
> >>> >>> "true" ) );
> >>> >>> +
> >>> >>> protected static ExecutorService executorService;
> >>> >>>
> >>> >>> + private static RingBuffer<PathTrackingEntry> RINGBUFFER;
> >>> >>> +
> >>> >>> static
> >>> >>> {
> >>> >>>
> >>> >>> @@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
> >>> >>> executorService = Executors.newFixedThreadPool(
> >>> threadsNumber
> >>> >>> );
> >>> >>>
> >>> >>> }
> >>> >>> +
> >>> >>> + if ( USE_DISRUPTOR )
> >>> >>> + {
> >>> >>> + ExecutorService exec =
> Executors.newCachedThreadPool();
> >>> >>> +
> >>> >>> + // FIXME make configurable: ring buffer size and
> >>> WaitStrategy
> >>> >>> +
> >>> >>> + Disruptor<PathTrackingEntry> disruptor =
> >>> >>> + new Disruptor<PathTrackingEntry>( new
> >>> >>> EventFactory<PathTrackingEntry>()
> >>> >>> + {
> >>> >>> + @Override
> >>> >>> + public PathTrackingEntry newInstance()
> >>> >>> + {
> >>> >>> + return new PathTrackingEntry();
> >>> >>> + }
> >>> >>> + }, 2048, exec, ProducerType.SINGLE, new
> >>> >>> BusySpinWaitStrategy()
> >>> >>> + );
> >>> >>> +
> >>> >>> + final EventHandler<PathTrackingEntry> handler = new
> >>> >>> EventHandler<PathTrackingEntry>()
> >>> >>> + {
> >>> >>> + // event will eventually be recycled by the
> Disruptor
> >>> >>> after it wraps
> >>> >>> + public void onEvent( final PathTrackingEntry
> entry,
> >>> >>> final long sequence, final boolean endOfBatch )
> >>> >>> + throws Exception
> >>> >>> + {
> >>> >>> + CUBE.postBytes( SerializeUtils.serialize(
> entry
> >>> >>> ), PathTrackingEntry.class.getName() );
> >>> >>> + }
> >>> >>> + };
> >>> >>> +
> >>> >>> + disruptor.handleEventsWith( handler );
> >>> >>> +
> >>> >>> + RINGBUFFER = disruptor.start();
> >>> >>> + }
> >>> >>> }
> >>> >>>
> >>> >>> @Override
> >>> >>> - public void store( PathTrackingEntry pathTrackingEntry )
> >>> >>> + public void store( final PathTrackingEntry pathTrackingEntry )
> >>> >>> {
> >>> >>> - cube.postBytes( SerializeUtils.serialize(
> pathTrackingEntry
> >>> >>> ), PathTrackingEntry.class.getName() );
> >>> >>> + if ( USE_DISRUPTOR )
> >>> >>> + {
> >>> >>> +
> >>> >>> + RINGBUFFER.publishEvent( new
> >>> >>> EventTranslator<PathTrackingEntry>()
> >>> >>> + {
> >>> >>> + @Override
> >>> >>> + public void translateTo( PathTrackingEntry event,
> >>> >>> long sequence )
> >>> >>> + {
> >>> >>> + event.setClassName(
> >>> pathTrackingEntry.getClassName()
> >>> >>> );
> >>> >>> + event.setExecutionTime(
> >>> >>> pathTrackingEntry.getExecutionTime() );
> >>> >>> + event.setLevel( pathTrackingEntry.getLevel()
> );
> >>> >>> + event.setMethodName(
> >>> >>> pathTrackingEntry.getMethodName() );
> >>> >>> + event.setNodeId(
> pathTrackingEntry.getNodeId() );
> >>> >>> + event.setStartTime(
> >>> pathTrackingEntry.getStartTime()
> >>> >>> );
> >>> >>> + event.setTrackingId(
> >>> >>> pathTrackingEntry.getTrackingId() );
> >>> >>> + }
> >>> >>> + } );
> >>> >>> +
> >>> >>> +
> >>> >>> + }
> >>> >>> + else
> >>> >>> + {
> >>> >>> + CUBE.postBytes( SerializeUtils.serialize(
> >>> >>> pathTrackingEntry ), PathTrackingEntry.class.getName() );
> >>> >>> + }
> >>> >>> +
> >>> >>> }
> >>> >>>
> >>> >>> @Override
> >>> >>> protected void pushEntriesByBatch( Map<String, List<Pointer>>
> >>> >>> pathTrackingEntries )
> >>> >>> {
> >>> >>> - if (!USE_SINGLE_STORE)
> >>> >>> + if ( !USE_SINGLE_STORE )
> >>> >>> {
> >>> >>>
> >>> >>> for ( Map.Entry<String, List<Pointer>> entry :
> >>> >>> pathTrackingEntries.entrySet() )
> >>> >>> @@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
> >>> >>> {
> >>> >>> if ( !pointer.isFree() )
> >>> >>> {
> >>> >>> - cube.postBytes( readBytes( pointer ),
> >>> >>> PathTrackingEntry.class.getName() );
> >>> >>> + CUBE.postBytes( readBytes( pointer ),
> >>> >>> PathTrackingEntry.class.getName() );
> >>> >>> pointer.freeMemory();
> >>> >>> }
> >>> >>> }
> >>> >>> @@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
> >>> >>> {
> >>> >>> executorService.shutdownNow();
> >>> >>> }
> >>> >>> +
> >>> >>> +
> >>> >>> }
> >>> >>>
> >>> >>> Modified:
> >>> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> >>> >>> URL:
> >>> >>>
> >>>
> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> >>> >>>
> >>> >>>
> >>>
> ==============================================================================
> >>> >>> ---
> >>> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> >>> >>> (original)
> >>> >>> +++
> >>> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> >>> >>> Fri Mar 7 06:07:56 2014
> >>> >>> @@ -59,7 +59,6 @@ public class PathTracker
> >>> >>> private static boolean USE_SINGLE_STORE =
> Boolean.parseBoolean(
> >>> >>> Configuration.getProperty(
> >>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
> >>> >>> "false" ) );
> >>> >>>
> >>> >>> -
> >>> >>> protected static ExecutorService executorService;
> >>> >>>
> >>> >>> static
> >>> >>>
> >>> >>> Modified:
> >>> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> >>> >>> URL:
> >>> >>>
> >>>
> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> >>> >>>
> >>> >>>
> >>>
> ==============================================================================
> >>> >>> ---
> >>> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> >>> >>> (original)
> >>> >>> +++
> >>> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> >>> >>> Fri Mar 7 06:07:56 2014
> >>> >>> @@ -63,6 +63,11 @@ public class PathTrackingEntry
> >>> >>> */
> >>> >>> private int level;
> >>> >>>
> >>> >>> + public PathTrackingEntry()
> >>> >>> + {
> >>> >>> + // no op
> >>> >>> + }
> >>> >>> +
> >>> >>> public PathTrackingEntry( String trackingId, String nodeId,
> >>> >>> String className, String methodName, long startTime,
> >>> >>> long executionTime, int level )
> >>> >>> {
> >>> >>>
> >>>
> >>>
> >>>
> >>> --
> >>> Olivier Lamy
> >>> Ecetera: http://ecetera.com.au
> >>> http://twitter.com/olamy | http://linkedin.com/in/olamy
> >>>
> >
> >
> >
> > --
> > Olivier Lamy
> > Ecetera: http://ecetera.com.au
> > http://twitter.com/olamy | http://linkedin.com/in/olamy
>
--
Olivier Lamy
Ecetera: http://ecetera.com.au
http://twitter.com/olamy | http://linkedin.com/in/olamy
Re: Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Posted by Romain Manni-Bucau <rm...@gmail.com>.
great
btw i don't get the store is not a solution, store doesn't mean
synchronous or so. We have a lot of async store actually.
I know you think I insist but "I reduce the memory": do you have
figures here too? Using Unsafe is quite impacting (since then you need
more mem than the one you define on the JVM etc so it makes config
harder to do) so I'd love to be able to justify it by numbers.
Romain Manni-Bucau
Twitter: @rmannibucau
Blog: http://rmannibucau.wordpress.com/
LinkedIn: http://fr.linkedin.com/in/rmannibucau
Github: https://github.com/rmannibucau
2014-03-11 12:26 GMT+01:00 Olivier Lamy <ol...@apache.org>:
> On 11 March 2014 17:07, Romain Manni-Bucau <rm...@gmail.com> wrote:
>> I dont doubt about boot but what I saw was that then ringbuffer can slow
>> down or loose message. Normally you configure path tracking to capture only
> I know but I don't want to filter (track everything :-) )
>> what you want and then the storage is done asynchronously so perf qhouldnt
>> be that bad. If it is we should just do more async (using the store). Well
>> get the drawbacks of queue but these are known and controllable.
>
> That's why the store is just not an acceptable solution. I reduce the
> memory by storing only bytes (serialization of the objet and send it
> as it to the collector).
> My goal is to have a sort of "streaming processing" of those entries
> (ideally no more local storage).
>
> Anyway I made disruptor optional and not default.
>
>>
>> Wdyt?
>> Le 10 mars 2014 23:16, "Olivier Lamy" <ol...@apache.org> a écrit :
>>
>>> Hi
>>> Apologize for delayed response. I was a bit off during the weekend.
>>>
>>> On 7 March 2014 20:17, Romain Manni-Bucau <rm...@gmail.com> wrote:
>>> > well if you have figures and say me disruptor is always 10 times
>>> > faster than default impl (regarding overall jvm perf and not only this
>>> > thread) I'll say: ok keep disruptor only but since i'm not sure it
>>> > changes a lot of things i think default shouldnt use disruptor and we
>>> > should stay simple and don't bring disruptor transitively
>>>
>>> I don't have any figures except starting a tomcat locally with path
>>> tracking activated now works with disruptor :-)
>>> I can make it optional it's a big problem but that's really more
>>> performant (again not mathematic figures just based on the fact it
>>> works better locally :-) )
>>>
>>>
>>> > Romain Manni-Bucau
>>> > Twitter: @rmannibucau
>>> > Blog: http://rmannibucau.wordpress.com/
>>> > LinkedIn: http://fr.linkedin.com/in/rmannibucau
>>> > Github: https://github.com/rmannibucau
>>> >
>>> >
>>> >
>>> > 2014-03-07 9:15 GMT+01:00 Jean-Louis MONTEIRO <je...@gmail.com>:
>>> >> Romain not sure I got the mandatory with 10.
>>> >> BTW big big thx you. Great feature.
>>> >> Gonna try to use that and get back to the project.
>>> >>
>>> >> Jean Louis
>>> >> Le 7 mars 2014 08:19, "Romain Manni-Bucau" <rm...@gmail.com> a
>>> écrit :
>>> >>
>>> >>> Hi
>>> >>>
>>> >>> any issue making disruptor optional and creating two
>>> >>> CubePathTrackingDataStore impl (the old one and disruptor one)? Think
>>> >>> we shouldn't make disruptor mandatory excepted if perf are > 10.
>>> >>>
>>> >>> Romain Manni-Bucau
>>> >>> Twitter: @rmannibucau
>>> >>> Blog: http://rmannibucau.wordpress.com/
>>> >>> LinkedIn: http://fr.linkedin.com/in/rmannibucau
>>> >>> Github: https://github.com/rmannibucau
>>> >>>
>>> >>>
>>> >>>
>>> >>>
>>> >>> ---------- Forwarded message ----------
>>> >>> From: <ol...@apache.org>
>>> >>> Date: 2014-03-07 7:07 GMT+01:00
>>> >>> Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
>>> >>> agent/store/cube/
>>> >>> agent/store/cube/src/main/java/org/apache/sirona/cube/
>>> >>> core/src/main/java/org/apache/sirona/tracking/
>>> >>> To: commits@sirona.incubator.apache.org
>>> >>>
>>> >>>
>>> >>> Author: olamy
>>> >>> Date: Fri Mar 7 06:07:56 2014
>>> >>> New Revision: 1575186
>>> >>>
>>> >>> URL: http://svn.apache.org/r1575186
>>> >>> Log:
>>> >>> use disruptor to handle amount of messages and not block the
>>> >>> application when sending path tracking entries
>>> >>>
>>> >>> Modified:
>>> >>> incubator/sirona/trunk/agent/store/cube/pom.xml
>>> >>>
>>> >>>
>>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>> >>>
>>> >>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>> >>>
>>> >>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>> >>>
>>> >>> Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
>>> >>> URL:
>>> >>>
>>> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
>>> >>>
>>> >>>
>>> ==============================================================================
>>> >>> --- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
>>> >>> +++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7 06:07:56
>>> >>> 2014
>>> >>> @@ -32,6 +32,13 @@
>>> >>> <dependency>
>>> >>> <groupId>org.apache.sirona</groupId>
>>> >>> <artifactId>sirona-core</artifactId>
>>> >>> + <scope>provided</scope>
>>> >>> + </dependency>
>>> >>> +
>>> >>> + <dependency>
>>> >>> + <groupId>com.lmax</groupId>
>>> >>> + <artifactId>disruptor</artifactId>
>>> >>> + <version>3.2.0</version>
>>> >>> </dependency>
>>> >>>
>>> >>> <dependency>
>>> >>> @@ -69,6 +76,34 @@
>>> >>> </execution>
>>> >>> </executions>
>>> >>> </plugin>
>>> >>> + <plugin>
>>> >>> + <groupId>org.apache.maven.plugins</groupId>
>>> >>> + <artifactId>maven-shade-plugin</artifactId>
>>> >>> + <executions>
>>> >>> + <execution>
>>> >>> + <goals>
>>> >>> + <goal>shade</goal>
>>> >>> + </goals>
>>> >>> + <configuration>
>>> >>> + <createSourcesJar>true</createSourcesJar>
>>> >>> + <shadedArtifactAttached>true</shadedArtifactAttached>
>>> >>> + <shadedClassifierName>shaded</shadedClassifierName>
>>> >>> + <relocations>
>>> >>> + <relocation>
>>> >>> + <pattern>com.lmax</pattern>
>>> >>> +
>>> >>> <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
>>> >>> + </relocation>
>>> >>> + <!--
>>> >>> + <relocation>
>>> >>> + <pattern>org.apache.sirona</pattern>
>>> >>> +
>>> <shadedPattern>org.apache.sirona.sirona</shadedPattern>
>>> >>> + </relocation>
>>> >>> + -->
>>> >>> + </relocations>
>>> >>> + </configuration>
>>> >>> + </execution>
>>> >>> + </executions>
>>> >>> + </plugin>
>>> >>> </plugins>
>>> >>> <pluginManagement>
>>> >>> <plugins>
>>> >>>
>>> >>> Modified:
>>> >>>
>>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>> >>> URL:
>>> >>>
>>> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>> >>>
>>> >>>
>>> ==============================================================================
>>> >>> ---
>>> >>>
>>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>> >>> (original)
>>> >>> +++
>>> >>>
>>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>> >>> Fri Mar 7 06:07:56 2014
>>> >>> @@ -17,6 +17,13 @@
>>> >>>
>>> >>> package org.apache.sirona.cube;
>>> >>>
>>> >>> +import com.lmax.disruptor.BusySpinWaitStrategy;
>>> >>> +import com.lmax.disruptor.EventFactory;
>>> >>> +import com.lmax.disruptor.EventHandler;
>>> >>> +import com.lmax.disruptor.EventTranslator;
>>> >>> +import com.lmax.disruptor.RingBuffer;
>>> >>> +import com.lmax.disruptor.dsl.Disruptor;
>>> >>> +import com.lmax.disruptor.dsl.ProducerType;
>>> >>> import org.apache.sirona.configuration.Configuration;
>>> >>> import org.apache.sirona.configuration.ioc.Destroying;
>>> >>> import org.apache.sirona.configuration.ioc.IoCs;
>>> >>> @@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
>>> >>> extends BatchPathTrackingDataStore
>>> >>> implements CollectorPathTrackingDataStore
>>> >>> {
>>> >>> - private final Cube cube = IoCs.findOrCreateInstance(
>>> >>> CubeBuilder.class ).build();
>>> >>> + private static final Cube CUBE = IoCs.findOrCreateInstance(
>>> >>> CubeBuilder.class ).build();
>>> >>>
>>> >>>
>>> >>> private static final boolean USE_EXECUTORS = Boolean.parseBoolean(
>>> >>> @@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
>>> >>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
>>> >>> Configuration.getProperty(
>>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
>>> >>> "false" ) );
>>> >>>
>>> >>> + private static final boolean USE_DISRUPTOR = Boolean.parseBoolean(
>>> >>> + Configuration.getProperty(
>>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.usedisruptor",
>>> >>> "true" ) );
>>> >>> +
>>> >>> protected static ExecutorService executorService;
>>> >>>
>>> >>> + private static RingBuffer<PathTrackingEntry> RINGBUFFER;
>>> >>> +
>>> >>> static
>>> >>> {
>>> >>>
>>> >>> @@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
>>> >>> executorService = Executors.newFixedThreadPool(
>>> threadsNumber
>>> >>> );
>>> >>>
>>> >>> }
>>> >>> +
>>> >>> + if ( USE_DISRUPTOR )
>>> >>> + {
>>> >>> + ExecutorService exec = Executors.newCachedThreadPool();
>>> >>> +
>>> >>> + // FIXME make configurable: ring buffer size and
>>> WaitStrategy
>>> >>> +
>>> >>> + Disruptor<PathTrackingEntry> disruptor =
>>> >>> + new Disruptor<PathTrackingEntry>( new
>>> >>> EventFactory<PathTrackingEntry>()
>>> >>> + {
>>> >>> + @Override
>>> >>> + public PathTrackingEntry newInstance()
>>> >>> + {
>>> >>> + return new PathTrackingEntry();
>>> >>> + }
>>> >>> + }, 2048, exec, ProducerType.SINGLE, new
>>> >>> BusySpinWaitStrategy()
>>> >>> + );
>>> >>> +
>>> >>> + final EventHandler<PathTrackingEntry> handler = new
>>> >>> EventHandler<PathTrackingEntry>()
>>> >>> + {
>>> >>> + // event will eventually be recycled by the Disruptor
>>> >>> after it wraps
>>> >>> + public void onEvent( final PathTrackingEntry entry,
>>> >>> final long sequence, final boolean endOfBatch )
>>> >>> + throws Exception
>>> >>> + {
>>> >>> + CUBE.postBytes( SerializeUtils.serialize( entry
>>> >>> ), PathTrackingEntry.class.getName() );
>>> >>> + }
>>> >>> + };
>>> >>> +
>>> >>> + disruptor.handleEventsWith( handler );
>>> >>> +
>>> >>> + RINGBUFFER = disruptor.start();
>>> >>> + }
>>> >>> }
>>> >>>
>>> >>> @Override
>>> >>> - public void store( PathTrackingEntry pathTrackingEntry )
>>> >>> + public void store( final PathTrackingEntry pathTrackingEntry )
>>> >>> {
>>> >>> - cube.postBytes( SerializeUtils.serialize( pathTrackingEntry
>>> >>> ), PathTrackingEntry.class.getName() );
>>> >>> + if ( USE_DISRUPTOR )
>>> >>> + {
>>> >>> +
>>> >>> + RINGBUFFER.publishEvent( new
>>> >>> EventTranslator<PathTrackingEntry>()
>>> >>> + {
>>> >>> + @Override
>>> >>> + public void translateTo( PathTrackingEntry event,
>>> >>> long sequence )
>>> >>> + {
>>> >>> + event.setClassName(
>>> pathTrackingEntry.getClassName()
>>> >>> );
>>> >>> + event.setExecutionTime(
>>> >>> pathTrackingEntry.getExecutionTime() );
>>> >>> + event.setLevel( pathTrackingEntry.getLevel() );
>>> >>> + event.setMethodName(
>>> >>> pathTrackingEntry.getMethodName() );
>>> >>> + event.setNodeId( pathTrackingEntry.getNodeId() );
>>> >>> + event.setStartTime(
>>> pathTrackingEntry.getStartTime()
>>> >>> );
>>> >>> + event.setTrackingId(
>>> >>> pathTrackingEntry.getTrackingId() );
>>> >>> + }
>>> >>> + } );
>>> >>> +
>>> >>> +
>>> >>> + }
>>> >>> + else
>>> >>> + {
>>> >>> + CUBE.postBytes( SerializeUtils.serialize(
>>> >>> pathTrackingEntry ), PathTrackingEntry.class.getName() );
>>> >>> + }
>>> >>> +
>>> >>> }
>>> >>>
>>> >>> @Override
>>> >>> protected void pushEntriesByBatch( Map<String, List<Pointer>>
>>> >>> pathTrackingEntries )
>>> >>> {
>>> >>> - if (!USE_SINGLE_STORE)
>>> >>> + if ( !USE_SINGLE_STORE )
>>> >>> {
>>> >>>
>>> >>> for ( Map.Entry<String, List<Pointer>> entry :
>>> >>> pathTrackingEntries.entrySet() )
>>> >>> @@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
>>> >>> {
>>> >>> if ( !pointer.isFree() )
>>> >>> {
>>> >>> - cube.postBytes( readBytes( pointer ),
>>> >>> PathTrackingEntry.class.getName() );
>>> >>> + CUBE.postBytes( readBytes( pointer ),
>>> >>> PathTrackingEntry.class.getName() );
>>> >>> pointer.freeMemory();
>>> >>> }
>>> >>> }
>>> >>> @@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
>>> >>> {
>>> >>> executorService.shutdownNow();
>>> >>> }
>>> >>> +
>>> >>> +
>>> >>> }
>>> >>>
>>> >>> Modified:
>>> >>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>> >>> URL:
>>> >>>
>>> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>> >>>
>>> >>>
>>> ==============================================================================
>>> >>> ---
>>> >>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>> >>> (original)
>>> >>> +++
>>> >>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>> >>> Fri Mar 7 06:07:56 2014
>>> >>> @@ -59,7 +59,6 @@ public class PathTracker
>>> >>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
>>> >>> Configuration.getProperty(
>>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
>>> >>> "false" ) );
>>> >>>
>>> >>> -
>>> >>> protected static ExecutorService executorService;
>>> >>>
>>> >>> static
>>> >>>
>>> >>> Modified:
>>> >>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>> >>> URL:
>>> >>>
>>> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>> >>>
>>> >>>
>>> ==============================================================================
>>> >>> ---
>>> >>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>> >>> (original)
>>> >>> +++
>>> >>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>> >>> Fri Mar 7 06:07:56 2014
>>> >>> @@ -63,6 +63,11 @@ public class PathTrackingEntry
>>> >>> */
>>> >>> private int level;
>>> >>>
>>> >>> + public PathTrackingEntry()
>>> >>> + {
>>> >>> + // no op
>>> >>> + }
>>> >>> +
>>> >>> public PathTrackingEntry( String trackingId, String nodeId,
>>> >>> String className, String methodName, long startTime,
>>> >>> long executionTime, int level )
>>> >>> {
>>> >>>
>>>
>>>
>>>
>>> --
>>> Olivier Lamy
>>> Ecetera: http://ecetera.com.au
>>> http://twitter.com/olamy | http://linkedin.com/in/olamy
>>>
>
>
>
> --
> Olivier Lamy
> Ecetera: http://ecetera.com.au
> http://twitter.com/olamy | http://linkedin.com/in/olamy
Re: Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Posted by Olivier Lamy <ol...@apache.org>.
On 11 March 2014 17:07, Romain Manni-Bucau <rm...@gmail.com> wrote:
> I dont doubt about boot but what I saw was that then ringbuffer can slow
> down or loose message. Normally you configure path tracking to capture only
I know but I don't want to filter (track everything :-) )
> what you want and then the storage is done asynchronously so perf qhouldnt
> be that bad. If it is we should just do more async (using the store). Well
> get the drawbacks of queue but these are known and controllable.
That's why the store is just not an acceptable solution. I reduce the
memory by storing only bytes (serialization of the objet and send it
as it to the collector).
My goal is to have a sort of "streaming processing" of those entries
(ideally no more local storage).
Anyway I made disruptor optional and not default.
>
> Wdyt?
> Le 10 mars 2014 23:16, "Olivier Lamy" <ol...@apache.org> a écrit :
>
>> Hi
>> Apologize for delayed response. I was a bit off during the weekend.
>>
>> On 7 March 2014 20:17, Romain Manni-Bucau <rm...@gmail.com> wrote:
>> > well if you have figures and say me disruptor is always 10 times
>> > faster than default impl (regarding overall jvm perf and not only this
>> > thread) I'll say: ok keep disruptor only but since i'm not sure it
>> > changes a lot of things i think default shouldnt use disruptor and we
>> > should stay simple and don't bring disruptor transitively
>>
>> I don't have any figures except starting a tomcat locally with path
>> tracking activated now works with disruptor :-)
>> I can make it optional it's a big problem but that's really more
>> performant (again not mathematic figures just based on the fact it
>> works better locally :-) )
>>
>>
>> > Romain Manni-Bucau
>> > Twitter: @rmannibucau
>> > Blog: http://rmannibucau.wordpress.com/
>> > LinkedIn: http://fr.linkedin.com/in/rmannibucau
>> > Github: https://github.com/rmannibucau
>> >
>> >
>> >
>> > 2014-03-07 9:15 GMT+01:00 Jean-Louis MONTEIRO <je...@gmail.com>:
>> >> Romain not sure I got the mandatory with 10.
>> >> BTW big big thx you. Great feature.
>> >> Gonna try to use that and get back to the project.
>> >>
>> >> Jean Louis
>> >> Le 7 mars 2014 08:19, "Romain Manni-Bucau" <rm...@gmail.com> a
>> écrit :
>> >>
>> >>> Hi
>> >>>
>> >>> any issue making disruptor optional and creating two
>> >>> CubePathTrackingDataStore impl (the old one and disruptor one)? Think
>> >>> we shouldn't make disruptor mandatory excepted if perf are > 10.
>> >>>
>> >>> Romain Manni-Bucau
>> >>> Twitter: @rmannibucau
>> >>> Blog: http://rmannibucau.wordpress.com/
>> >>> LinkedIn: http://fr.linkedin.com/in/rmannibucau
>> >>> Github: https://github.com/rmannibucau
>> >>>
>> >>>
>> >>>
>> >>>
>> >>> ---------- Forwarded message ----------
>> >>> From: <ol...@apache.org>
>> >>> Date: 2014-03-07 7:07 GMT+01:00
>> >>> Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
>> >>> agent/store/cube/
>> >>> agent/store/cube/src/main/java/org/apache/sirona/cube/
>> >>> core/src/main/java/org/apache/sirona/tracking/
>> >>> To: commits@sirona.incubator.apache.org
>> >>>
>> >>>
>> >>> Author: olamy
>> >>> Date: Fri Mar 7 06:07:56 2014
>> >>> New Revision: 1575186
>> >>>
>> >>> URL: http://svn.apache.org/r1575186
>> >>> Log:
>> >>> use disruptor to handle amount of messages and not block the
>> >>> application when sending path tracking entries
>> >>>
>> >>> Modified:
>> >>> incubator/sirona/trunk/agent/store/cube/pom.xml
>> >>>
>> >>>
>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>> >>>
>> >>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>> >>>
>> >>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>> >>>
>> >>> Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
>> >>> URL:
>> >>>
>> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
>> >>>
>> >>>
>> ==============================================================================
>> >>> --- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
>> >>> +++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7 06:07:56
>> >>> 2014
>> >>> @@ -32,6 +32,13 @@
>> >>> <dependency>
>> >>> <groupId>org.apache.sirona</groupId>
>> >>> <artifactId>sirona-core</artifactId>
>> >>> + <scope>provided</scope>
>> >>> + </dependency>
>> >>> +
>> >>> + <dependency>
>> >>> + <groupId>com.lmax</groupId>
>> >>> + <artifactId>disruptor</artifactId>
>> >>> + <version>3.2.0</version>
>> >>> </dependency>
>> >>>
>> >>> <dependency>
>> >>> @@ -69,6 +76,34 @@
>> >>> </execution>
>> >>> </executions>
>> >>> </plugin>
>> >>> + <plugin>
>> >>> + <groupId>org.apache.maven.plugins</groupId>
>> >>> + <artifactId>maven-shade-plugin</artifactId>
>> >>> + <executions>
>> >>> + <execution>
>> >>> + <goals>
>> >>> + <goal>shade</goal>
>> >>> + </goals>
>> >>> + <configuration>
>> >>> + <createSourcesJar>true</createSourcesJar>
>> >>> + <shadedArtifactAttached>true</shadedArtifactAttached>
>> >>> + <shadedClassifierName>shaded</shadedClassifierName>
>> >>> + <relocations>
>> >>> + <relocation>
>> >>> + <pattern>com.lmax</pattern>
>> >>> +
>> >>> <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
>> >>> + </relocation>
>> >>> + <!--
>> >>> + <relocation>
>> >>> + <pattern>org.apache.sirona</pattern>
>> >>> +
>> <shadedPattern>org.apache.sirona.sirona</shadedPattern>
>> >>> + </relocation>
>> >>> + -->
>> >>> + </relocations>
>> >>> + </configuration>
>> >>> + </execution>
>> >>> + </executions>
>> >>> + </plugin>
>> >>> </plugins>
>> >>> <pluginManagement>
>> >>> <plugins>
>> >>>
>> >>> Modified:
>> >>>
>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>> >>> URL:
>> >>>
>> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>> >>>
>> >>>
>> ==============================================================================
>> >>> ---
>> >>>
>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>> >>> (original)
>> >>> +++
>> >>>
>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>> >>> Fri Mar 7 06:07:56 2014
>> >>> @@ -17,6 +17,13 @@
>> >>>
>> >>> package org.apache.sirona.cube;
>> >>>
>> >>> +import com.lmax.disruptor.BusySpinWaitStrategy;
>> >>> +import com.lmax.disruptor.EventFactory;
>> >>> +import com.lmax.disruptor.EventHandler;
>> >>> +import com.lmax.disruptor.EventTranslator;
>> >>> +import com.lmax.disruptor.RingBuffer;
>> >>> +import com.lmax.disruptor.dsl.Disruptor;
>> >>> +import com.lmax.disruptor.dsl.ProducerType;
>> >>> import org.apache.sirona.configuration.Configuration;
>> >>> import org.apache.sirona.configuration.ioc.Destroying;
>> >>> import org.apache.sirona.configuration.ioc.IoCs;
>> >>> @@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
>> >>> extends BatchPathTrackingDataStore
>> >>> implements CollectorPathTrackingDataStore
>> >>> {
>> >>> - private final Cube cube = IoCs.findOrCreateInstance(
>> >>> CubeBuilder.class ).build();
>> >>> + private static final Cube CUBE = IoCs.findOrCreateInstance(
>> >>> CubeBuilder.class ).build();
>> >>>
>> >>>
>> >>> private static final boolean USE_EXECUTORS = Boolean.parseBoolean(
>> >>> @@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
>> >>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
>> >>> Configuration.getProperty(
>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
>> >>> "false" ) );
>> >>>
>> >>> + private static final boolean USE_DISRUPTOR = Boolean.parseBoolean(
>> >>> + Configuration.getProperty(
>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.usedisruptor",
>> >>> "true" ) );
>> >>> +
>> >>> protected static ExecutorService executorService;
>> >>>
>> >>> + private static RingBuffer<PathTrackingEntry> RINGBUFFER;
>> >>> +
>> >>> static
>> >>> {
>> >>>
>> >>> @@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
>> >>> executorService = Executors.newFixedThreadPool(
>> threadsNumber
>> >>> );
>> >>>
>> >>> }
>> >>> +
>> >>> + if ( USE_DISRUPTOR )
>> >>> + {
>> >>> + ExecutorService exec = Executors.newCachedThreadPool();
>> >>> +
>> >>> + // FIXME make configurable: ring buffer size and
>> WaitStrategy
>> >>> +
>> >>> + Disruptor<PathTrackingEntry> disruptor =
>> >>> + new Disruptor<PathTrackingEntry>( new
>> >>> EventFactory<PathTrackingEntry>()
>> >>> + {
>> >>> + @Override
>> >>> + public PathTrackingEntry newInstance()
>> >>> + {
>> >>> + return new PathTrackingEntry();
>> >>> + }
>> >>> + }, 2048, exec, ProducerType.SINGLE, new
>> >>> BusySpinWaitStrategy()
>> >>> + );
>> >>> +
>> >>> + final EventHandler<PathTrackingEntry> handler = new
>> >>> EventHandler<PathTrackingEntry>()
>> >>> + {
>> >>> + // event will eventually be recycled by the Disruptor
>> >>> after it wraps
>> >>> + public void onEvent( final PathTrackingEntry entry,
>> >>> final long sequence, final boolean endOfBatch )
>> >>> + throws Exception
>> >>> + {
>> >>> + CUBE.postBytes( SerializeUtils.serialize( entry
>> >>> ), PathTrackingEntry.class.getName() );
>> >>> + }
>> >>> + };
>> >>> +
>> >>> + disruptor.handleEventsWith( handler );
>> >>> +
>> >>> + RINGBUFFER = disruptor.start();
>> >>> + }
>> >>> }
>> >>>
>> >>> @Override
>> >>> - public void store( PathTrackingEntry pathTrackingEntry )
>> >>> + public void store( final PathTrackingEntry pathTrackingEntry )
>> >>> {
>> >>> - cube.postBytes( SerializeUtils.serialize( pathTrackingEntry
>> >>> ), PathTrackingEntry.class.getName() );
>> >>> + if ( USE_DISRUPTOR )
>> >>> + {
>> >>> +
>> >>> + RINGBUFFER.publishEvent( new
>> >>> EventTranslator<PathTrackingEntry>()
>> >>> + {
>> >>> + @Override
>> >>> + public void translateTo( PathTrackingEntry event,
>> >>> long sequence )
>> >>> + {
>> >>> + event.setClassName(
>> pathTrackingEntry.getClassName()
>> >>> );
>> >>> + event.setExecutionTime(
>> >>> pathTrackingEntry.getExecutionTime() );
>> >>> + event.setLevel( pathTrackingEntry.getLevel() );
>> >>> + event.setMethodName(
>> >>> pathTrackingEntry.getMethodName() );
>> >>> + event.setNodeId( pathTrackingEntry.getNodeId() );
>> >>> + event.setStartTime(
>> pathTrackingEntry.getStartTime()
>> >>> );
>> >>> + event.setTrackingId(
>> >>> pathTrackingEntry.getTrackingId() );
>> >>> + }
>> >>> + } );
>> >>> +
>> >>> +
>> >>> + }
>> >>> + else
>> >>> + {
>> >>> + CUBE.postBytes( SerializeUtils.serialize(
>> >>> pathTrackingEntry ), PathTrackingEntry.class.getName() );
>> >>> + }
>> >>> +
>> >>> }
>> >>>
>> >>> @Override
>> >>> protected void pushEntriesByBatch( Map<String, List<Pointer>>
>> >>> pathTrackingEntries )
>> >>> {
>> >>> - if (!USE_SINGLE_STORE)
>> >>> + if ( !USE_SINGLE_STORE )
>> >>> {
>> >>>
>> >>> for ( Map.Entry<String, List<Pointer>> entry :
>> >>> pathTrackingEntries.entrySet() )
>> >>> @@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
>> >>> {
>> >>> if ( !pointer.isFree() )
>> >>> {
>> >>> - cube.postBytes( readBytes( pointer ),
>> >>> PathTrackingEntry.class.getName() );
>> >>> + CUBE.postBytes( readBytes( pointer ),
>> >>> PathTrackingEntry.class.getName() );
>> >>> pointer.freeMemory();
>> >>> }
>> >>> }
>> >>> @@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
>> >>> {
>> >>> executorService.shutdownNow();
>> >>> }
>> >>> +
>> >>> +
>> >>> }
>> >>>
>> >>> Modified:
>> >>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>> >>> URL:
>> >>>
>> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>> >>>
>> >>>
>> ==============================================================================
>> >>> ---
>> >>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>> >>> (original)
>> >>> +++
>> >>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>> >>> Fri Mar 7 06:07:56 2014
>> >>> @@ -59,7 +59,6 @@ public class PathTracker
>> >>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
>> >>> Configuration.getProperty(
>> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
>> >>> "false" ) );
>> >>>
>> >>> -
>> >>> protected static ExecutorService executorService;
>> >>>
>> >>> static
>> >>>
>> >>> Modified:
>> >>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>> >>> URL:
>> >>>
>> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>> >>>
>> >>>
>> ==============================================================================
>> >>> ---
>> >>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>> >>> (original)
>> >>> +++
>> >>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>> >>> Fri Mar 7 06:07:56 2014
>> >>> @@ -63,6 +63,11 @@ public class PathTrackingEntry
>> >>> */
>> >>> private int level;
>> >>>
>> >>> + public PathTrackingEntry()
>> >>> + {
>> >>> + // no op
>> >>> + }
>> >>> +
>> >>> public PathTrackingEntry( String trackingId, String nodeId,
>> >>> String className, String methodName, long startTime,
>> >>> long executionTime, int level )
>> >>> {
>> >>>
>>
>>
>>
>> --
>> Olivier Lamy
>> Ecetera: http://ecetera.com.au
>> http://twitter.com/olamy | http://linkedin.com/in/olamy
>>
--
Olivier Lamy
Ecetera: http://ecetera.com.au
http://twitter.com/olamy | http://linkedin.com/in/olamy
Re: Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Posted by Romain Manni-Bucau <rm...@gmail.com>.
I dont doubt about boot but what I saw was that then ringbuffer can slow
down or loose message. Normally you configure path tracking to capture only
what you want and then the storage is done asynchronously so perf qhouldnt
be that bad. If it is we should just do more async (using the store). Well
get the drawbacks of queue but these are known and controllable.
Wdyt?
Le 10 mars 2014 23:16, "Olivier Lamy" <ol...@apache.org> a écrit :
> Hi
> Apologize for delayed response. I was a bit off during the weekend.
>
> On 7 March 2014 20:17, Romain Manni-Bucau <rm...@gmail.com> wrote:
> > well if you have figures and say me disruptor is always 10 times
> > faster than default impl (regarding overall jvm perf and not only this
> > thread) I'll say: ok keep disruptor only but since i'm not sure it
> > changes a lot of things i think default shouldnt use disruptor and we
> > should stay simple and don't bring disruptor transitively
>
> I don't have any figures except starting a tomcat locally with path
> tracking activated now works with disruptor :-)
> I can make it optional it's a big problem but that's really more
> performant (again not mathematic figures just based on the fact it
> works better locally :-) )
>
>
> > Romain Manni-Bucau
> > Twitter: @rmannibucau
> > Blog: http://rmannibucau.wordpress.com/
> > LinkedIn: http://fr.linkedin.com/in/rmannibucau
> > Github: https://github.com/rmannibucau
> >
> >
> >
> > 2014-03-07 9:15 GMT+01:00 Jean-Louis MONTEIRO <je...@gmail.com>:
> >> Romain not sure I got the mandatory with 10.
> >> BTW big big thx you. Great feature.
> >> Gonna try to use that and get back to the project.
> >>
> >> Jean Louis
> >> Le 7 mars 2014 08:19, "Romain Manni-Bucau" <rm...@gmail.com> a
> écrit :
> >>
> >>> Hi
> >>>
> >>> any issue making disruptor optional and creating two
> >>> CubePathTrackingDataStore impl (the old one and disruptor one)? Think
> >>> we shouldn't make disruptor mandatory excepted if perf are > 10.
> >>>
> >>> Romain Manni-Bucau
> >>> Twitter: @rmannibucau
> >>> Blog: http://rmannibucau.wordpress.com/
> >>> LinkedIn: http://fr.linkedin.com/in/rmannibucau
> >>> Github: https://github.com/rmannibucau
> >>>
> >>>
> >>>
> >>>
> >>> ---------- Forwarded message ----------
> >>> From: <ol...@apache.org>
> >>> Date: 2014-03-07 7:07 GMT+01:00
> >>> Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
> >>> agent/store/cube/
> >>> agent/store/cube/src/main/java/org/apache/sirona/cube/
> >>> core/src/main/java/org/apache/sirona/tracking/
> >>> To: commits@sirona.incubator.apache.org
> >>>
> >>>
> >>> Author: olamy
> >>> Date: Fri Mar 7 06:07:56 2014
> >>> New Revision: 1575186
> >>>
> >>> URL: http://svn.apache.org/r1575186
> >>> Log:
> >>> use disruptor to handle amount of messages and not block the
> >>> application when sending path tracking entries
> >>>
> >>> Modified:
> >>> incubator/sirona/trunk/agent/store/cube/pom.xml
> >>>
> >>>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> >>>
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> >>>
> >>> Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
> >>> URL:
> >>>
> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
> >>>
> >>>
> ==============================================================================
> >>> --- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
> >>> +++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7 06:07:56
> >>> 2014
> >>> @@ -32,6 +32,13 @@
> >>> <dependency>
> >>> <groupId>org.apache.sirona</groupId>
> >>> <artifactId>sirona-core</artifactId>
> >>> + <scope>provided</scope>
> >>> + </dependency>
> >>> +
> >>> + <dependency>
> >>> + <groupId>com.lmax</groupId>
> >>> + <artifactId>disruptor</artifactId>
> >>> + <version>3.2.0</version>
> >>> </dependency>
> >>>
> >>> <dependency>
> >>> @@ -69,6 +76,34 @@
> >>> </execution>
> >>> </executions>
> >>> </plugin>
> >>> + <plugin>
> >>> + <groupId>org.apache.maven.plugins</groupId>
> >>> + <artifactId>maven-shade-plugin</artifactId>
> >>> + <executions>
> >>> + <execution>
> >>> + <goals>
> >>> + <goal>shade</goal>
> >>> + </goals>
> >>> + <configuration>
> >>> + <createSourcesJar>true</createSourcesJar>
> >>> + <shadedArtifactAttached>true</shadedArtifactAttached>
> >>> + <shadedClassifierName>shaded</shadedClassifierName>
> >>> + <relocations>
> >>> + <relocation>
> >>> + <pattern>com.lmax</pattern>
> >>> +
> >>> <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
> >>> + </relocation>
> >>> + <!--
> >>> + <relocation>
> >>> + <pattern>org.apache.sirona</pattern>
> >>> +
> <shadedPattern>org.apache.sirona.sirona</shadedPattern>
> >>> + </relocation>
> >>> + -->
> >>> + </relocations>
> >>> + </configuration>
> >>> + </execution>
> >>> + </executions>
> >>> + </plugin>
> >>> </plugins>
> >>> <pluginManagement>
> >>> <plugins>
> >>>
> >>> Modified:
> >>>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> >>> URL:
> >>>
> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> >>>
> >>>
> ==============================================================================
> >>> ---
> >>>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> >>> (original)
> >>> +++
> >>>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> >>> Fri Mar 7 06:07:56 2014
> >>> @@ -17,6 +17,13 @@
> >>>
> >>> package org.apache.sirona.cube;
> >>>
> >>> +import com.lmax.disruptor.BusySpinWaitStrategy;
> >>> +import com.lmax.disruptor.EventFactory;
> >>> +import com.lmax.disruptor.EventHandler;
> >>> +import com.lmax.disruptor.EventTranslator;
> >>> +import com.lmax.disruptor.RingBuffer;
> >>> +import com.lmax.disruptor.dsl.Disruptor;
> >>> +import com.lmax.disruptor.dsl.ProducerType;
> >>> import org.apache.sirona.configuration.Configuration;
> >>> import org.apache.sirona.configuration.ioc.Destroying;
> >>> import org.apache.sirona.configuration.ioc.IoCs;
> >>> @@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
> >>> extends BatchPathTrackingDataStore
> >>> implements CollectorPathTrackingDataStore
> >>> {
> >>> - private final Cube cube = IoCs.findOrCreateInstance(
> >>> CubeBuilder.class ).build();
> >>> + private static final Cube CUBE = IoCs.findOrCreateInstance(
> >>> CubeBuilder.class ).build();
> >>>
> >>>
> >>> private static final boolean USE_EXECUTORS = Boolean.parseBoolean(
> >>> @@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
> >>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
> >>> Configuration.getProperty(
> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
> >>> "false" ) );
> >>>
> >>> + private static final boolean USE_DISRUPTOR = Boolean.parseBoolean(
> >>> + Configuration.getProperty(
> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.usedisruptor",
> >>> "true" ) );
> >>> +
> >>> protected static ExecutorService executorService;
> >>>
> >>> + private static RingBuffer<PathTrackingEntry> RINGBUFFER;
> >>> +
> >>> static
> >>> {
> >>>
> >>> @@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
> >>> executorService = Executors.newFixedThreadPool(
> threadsNumber
> >>> );
> >>>
> >>> }
> >>> +
> >>> + if ( USE_DISRUPTOR )
> >>> + {
> >>> + ExecutorService exec = Executors.newCachedThreadPool();
> >>> +
> >>> + // FIXME make configurable: ring buffer size and
> WaitStrategy
> >>> +
> >>> + Disruptor<PathTrackingEntry> disruptor =
> >>> + new Disruptor<PathTrackingEntry>( new
> >>> EventFactory<PathTrackingEntry>()
> >>> + {
> >>> + @Override
> >>> + public PathTrackingEntry newInstance()
> >>> + {
> >>> + return new PathTrackingEntry();
> >>> + }
> >>> + }, 2048, exec, ProducerType.SINGLE, new
> >>> BusySpinWaitStrategy()
> >>> + );
> >>> +
> >>> + final EventHandler<PathTrackingEntry> handler = new
> >>> EventHandler<PathTrackingEntry>()
> >>> + {
> >>> + // event will eventually be recycled by the Disruptor
> >>> after it wraps
> >>> + public void onEvent( final PathTrackingEntry entry,
> >>> final long sequence, final boolean endOfBatch )
> >>> + throws Exception
> >>> + {
> >>> + CUBE.postBytes( SerializeUtils.serialize( entry
> >>> ), PathTrackingEntry.class.getName() );
> >>> + }
> >>> + };
> >>> +
> >>> + disruptor.handleEventsWith( handler );
> >>> +
> >>> + RINGBUFFER = disruptor.start();
> >>> + }
> >>> }
> >>>
> >>> @Override
> >>> - public void store( PathTrackingEntry pathTrackingEntry )
> >>> + public void store( final PathTrackingEntry pathTrackingEntry )
> >>> {
> >>> - cube.postBytes( SerializeUtils.serialize( pathTrackingEntry
> >>> ), PathTrackingEntry.class.getName() );
> >>> + if ( USE_DISRUPTOR )
> >>> + {
> >>> +
> >>> + RINGBUFFER.publishEvent( new
> >>> EventTranslator<PathTrackingEntry>()
> >>> + {
> >>> + @Override
> >>> + public void translateTo( PathTrackingEntry event,
> >>> long sequence )
> >>> + {
> >>> + event.setClassName(
> pathTrackingEntry.getClassName()
> >>> );
> >>> + event.setExecutionTime(
> >>> pathTrackingEntry.getExecutionTime() );
> >>> + event.setLevel( pathTrackingEntry.getLevel() );
> >>> + event.setMethodName(
> >>> pathTrackingEntry.getMethodName() );
> >>> + event.setNodeId( pathTrackingEntry.getNodeId() );
> >>> + event.setStartTime(
> pathTrackingEntry.getStartTime()
> >>> );
> >>> + event.setTrackingId(
> >>> pathTrackingEntry.getTrackingId() );
> >>> + }
> >>> + } );
> >>> +
> >>> +
> >>> + }
> >>> + else
> >>> + {
> >>> + CUBE.postBytes( SerializeUtils.serialize(
> >>> pathTrackingEntry ), PathTrackingEntry.class.getName() );
> >>> + }
> >>> +
> >>> }
> >>>
> >>> @Override
> >>> protected void pushEntriesByBatch( Map<String, List<Pointer>>
> >>> pathTrackingEntries )
> >>> {
> >>> - if (!USE_SINGLE_STORE)
> >>> + if ( !USE_SINGLE_STORE )
> >>> {
> >>>
> >>> for ( Map.Entry<String, List<Pointer>> entry :
> >>> pathTrackingEntries.entrySet() )
> >>> @@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
> >>> {
> >>> if ( !pointer.isFree() )
> >>> {
> >>> - cube.postBytes( readBytes( pointer ),
> >>> PathTrackingEntry.class.getName() );
> >>> + CUBE.postBytes( readBytes( pointer ),
> >>> PathTrackingEntry.class.getName() );
> >>> pointer.freeMemory();
> >>> }
> >>> }
> >>> @@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
> >>> {
> >>> executorService.shutdownNow();
> >>> }
> >>> +
> >>> +
> >>> }
> >>>
> >>> Modified:
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> >>> URL:
> >>>
> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> >>>
> >>>
> ==============================================================================
> >>> ---
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> >>> (original)
> >>> +++
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> >>> Fri Mar 7 06:07:56 2014
> >>> @@ -59,7 +59,6 @@ public class PathTracker
> >>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
> >>> Configuration.getProperty(
> >>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
> >>> "false" ) );
> >>>
> >>> -
> >>> protected static ExecutorService executorService;
> >>>
> >>> static
> >>>
> >>> Modified:
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> >>> URL:
> >>>
> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
> >>>
> >>>
> ==============================================================================
> >>> ---
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> >>> (original)
> >>> +++
> >>>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> >>> Fri Mar 7 06:07:56 2014
> >>> @@ -63,6 +63,11 @@ public class PathTrackingEntry
> >>> */
> >>> private int level;
> >>>
> >>> + public PathTrackingEntry()
> >>> + {
> >>> + // no op
> >>> + }
> >>> +
> >>> public PathTrackingEntry( String trackingId, String nodeId,
> >>> String className, String methodName, long startTime,
> >>> long executionTime, int level )
> >>> {
> >>>
>
>
>
> --
> Olivier Lamy
> Ecetera: http://ecetera.com.au
> http://twitter.com/olamy | http://linkedin.com/in/olamy
>
Re: Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Posted by Olivier Lamy <ol...@apache.org>.
Hi
Apologize for delayed response. I was a bit off during the weekend.
On 7 March 2014 20:17, Romain Manni-Bucau <rm...@gmail.com> wrote:
> well if you have figures and say me disruptor is always 10 times
> faster than default impl (regarding overall jvm perf and not only this
> thread) I'll say: ok keep disruptor only but since i'm not sure it
> changes a lot of things i think default shouldnt use disruptor and we
> should stay simple and don't bring disruptor transitively
I don't have any figures except starting a tomcat locally with path
tracking activated now works with disruptor :-)
I can make it optional it's a big problem but that's really more
performant (again not mathematic figures just based on the fact it
works better locally :-) )
> Romain Manni-Bucau
> Twitter: @rmannibucau
> Blog: http://rmannibucau.wordpress.com/
> LinkedIn: http://fr.linkedin.com/in/rmannibucau
> Github: https://github.com/rmannibucau
>
>
>
> 2014-03-07 9:15 GMT+01:00 Jean-Louis MONTEIRO <je...@gmail.com>:
>> Romain not sure I got the mandatory with 10.
>> BTW big big thx you. Great feature.
>> Gonna try to use that and get back to the project.
>>
>> Jean Louis
>> Le 7 mars 2014 08:19, "Romain Manni-Bucau" <rm...@gmail.com> a écrit :
>>
>>> Hi
>>>
>>> any issue making disruptor optional and creating two
>>> CubePathTrackingDataStore impl (the old one and disruptor one)? Think
>>> we shouldn't make disruptor mandatory excepted if perf are > 10.
>>>
>>> Romain Manni-Bucau
>>> Twitter: @rmannibucau
>>> Blog: http://rmannibucau.wordpress.com/
>>> LinkedIn: http://fr.linkedin.com/in/rmannibucau
>>> Github: https://github.com/rmannibucau
>>>
>>>
>>>
>>>
>>> ---------- Forwarded message ----------
>>> From: <ol...@apache.org>
>>> Date: 2014-03-07 7:07 GMT+01:00
>>> Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
>>> agent/store/cube/
>>> agent/store/cube/src/main/java/org/apache/sirona/cube/
>>> core/src/main/java/org/apache/sirona/tracking/
>>> To: commits@sirona.incubator.apache.org
>>>
>>>
>>> Author: olamy
>>> Date: Fri Mar 7 06:07:56 2014
>>> New Revision: 1575186
>>>
>>> URL: http://svn.apache.org/r1575186
>>> Log:
>>> use disruptor to handle amount of messages and not block the
>>> application when sending path tracking entries
>>>
>>> Modified:
>>> incubator/sirona/trunk/agent/store/cube/pom.xml
>>>
>>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>>
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>>
>>> Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
>>> URL:
>>> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
>>>
>>> ==============================================================================
>>> --- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
>>> +++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7 06:07:56
>>> 2014
>>> @@ -32,6 +32,13 @@
>>> <dependency>
>>> <groupId>org.apache.sirona</groupId>
>>> <artifactId>sirona-core</artifactId>
>>> + <scope>provided</scope>
>>> + </dependency>
>>> +
>>> + <dependency>
>>> + <groupId>com.lmax</groupId>
>>> + <artifactId>disruptor</artifactId>
>>> + <version>3.2.0</version>
>>> </dependency>
>>>
>>> <dependency>
>>> @@ -69,6 +76,34 @@
>>> </execution>
>>> </executions>
>>> </plugin>
>>> + <plugin>
>>> + <groupId>org.apache.maven.plugins</groupId>
>>> + <artifactId>maven-shade-plugin</artifactId>
>>> + <executions>
>>> + <execution>
>>> + <goals>
>>> + <goal>shade</goal>
>>> + </goals>
>>> + <configuration>
>>> + <createSourcesJar>true</createSourcesJar>
>>> + <shadedArtifactAttached>true</shadedArtifactAttached>
>>> + <shadedClassifierName>shaded</shadedClassifierName>
>>> + <relocations>
>>> + <relocation>
>>> + <pattern>com.lmax</pattern>
>>> +
>>> <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
>>> + </relocation>
>>> + <!--
>>> + <relocation>
>>> + <pattern>org.apache.sirona</pattern>
>>> + <shadedPattern>org.apache.sirona.sirona</shadedPattern>
>>> + </relocation>
>>> + -->
>>> + </relocations>
>>> + </configuration>
>>> + </execution>
>>> + </executions>
>>> + </plugin>
>>> </plugins>
>>> <pluginManagement>
>>> <plugins>
>>>
>>> Modified:
>>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>> URL:
>>> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>> (original)
>>> +++
>>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>> Fri Mar 7 06:07:56 2014
>>> @@ -17,6 +17,13 @@
>>>
>>> package org.apache.sirona.cube;
>>>
>>> +import com.lmax.disruptor.BusySpinWaitStrategy;
>>> +import com.lmax.disruptor.EventFactory;
>>> +import com.lmax.disruptor.EventHandler;
>>> +import com.lmax.disruptor.EventTranslator;
>>> +import com.lmax.disruptor.RingBuffer;
>>> +import com.lmax.disruptor.dsl.Disruptor;
>>> +import com.lmax.disruptor.dsl.ProducerType;
>>> import org.apache.sirona.configuration.Configuration;
>>> import org.apache.sirona.configuration.ioc.Destroying;
>>> import org.apache.sirona.configuration.ioc.IoCs;
>>> @@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
>>> extends BatchPathTrackingDataStore
>>> implements CollectorPathTrackingDataStore
>>> {
>>> - private final Cube cube = IoCs.findOrCreateInstance(
>>> CubeBuilder.class ).build();
>>> + private static final Cube CUBE = IoCs.findOrCreateInstance(
>>> CubeBuilder.class ).build();
>>>
>>>
>>> private static final boolean USE_EXECUTORS = Boolean.parseBoolean(
>>> @@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
>>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
>>> Configuration.getProperty(
>>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
>>> "false" ) );
>>>
>>> + private static final boolean USE_DISRUPTOR = Boolean.parseBoolean(
>>> + Configuration.getProperty(
>>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.usedisruptor",
>>> "true" ) );
>>> +
>>> protected static ExecutorService executorService;
>>>
>>> + private static RingBuffer<PathTrackingEntry> RINGBUFFER;
>>> +
>>> static
>>> {
>>>
>>> @@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
>>> executorService = Executors.newFixedThreadPool( threadsNumber
>>> );
>>>
>>> }
>>> +
>>> + if ( USE_DISRUPTOR )
>>> + {
>>> + ExecutorService exec = Executors.newCachedThreadPool();
>>> +
>>> + // FIXME make configurable: ring buffer size and WaitStrategy
>>> +
>>> + Disruptor<PathTrackingEntry> disruptor =
>>> + new Disruptor<PathTrackingEntry>( new
>>> EventFactory<PathTrackingEntry>()
>>> + {
>>> + @Override
>>> + public PathTrackingEntry newInstance()
>>> + {
>>> + return new PathTrackingEntry();
>>> + }
>>> + }, 2048, exec, ProducerType.SINGLE, new
>>> BusySpinWaitStrategy()
>>> + );
>>> +
>>> + final EventHandler<PathTrackingEntry> handler = new
>>> EventHandler<PathTrackingEntry>()
>>> + {
>>> + // event will eventually be recycled by the Disruptor
>>> after it wraps
>>> + public void onEvent( final PathTrackingEntry entry,
>>> final long sequence, final boolean endOfBatch )
>>> + throws Exception
>>> + {
>>> + CUBE.postBytes( SerializeUtils.serialize( entry
>>> ), PathTrackingEntry.class.getName() );
>>> + }
>>> + };
>>> +
>>> + disruptor.handleEventsWith( handler );
>>> +
>>> + RINGBUFFER = disruptor.start();
>>> + }
>>> }
>>>
>>> @Override
>>> - public void store( PathTrackingEntry pathTrackingEntry )
>>> + public void store( final PathTrackingEntry pathTrackingEntry )
>>> {
>>> - cube.postBytes( SerializeUtils.serialize( pathTrackingEntry
>>> ), PathTrackingEntry.class.getName() );
>>> + if ( USE_DISRUPTOR )
>>> + {
>>> +
>>> + RINGBUFFER.publishEvent( new
>>> EventTranslator<PathTrackingEntry>()
>>> + {
>>> + @Override
>>> + public void translateTo( PathTrackingEntry event,
>>> long sequence )
>>> + {
>>> + event.setClassName( pathTrackingEntry.getClassName()
>>> );
>>> + event.setExecutionTime(
>>> pathTrackingEntry.getExecutionTime() );
>>> + event.setLevel( pathTrackingEntry.getLevel() );
>>> + event.setMethodName(
>>> pathTrackingEntry.getMethodName() );
>>> + event.setNodeId( pathTrackingEntry.getNodeId() );
>>> + event.setStartTime( pathTrackingEntry.getStartTime()
>>> );
>>> + event.setTrackingId(
>>> pathTrackingEntry.getTrackingId() );
>>> + }
>>> + } );
>>> +
>>> +
>>> + }
>>> + else
>>> + {
>>> + CUBE.postBytes( SerializeUtils.serialize(
>>> pathTrackingEntry ), PathTrackingEntry.class.getName() );
>>> + }
>>> +
>>> }
>>>
>>> @Override
>>> protected void pushEntriesByBatch( Map<String, List<Pointer>>
>>> pathTrackingEntries )
>>> {
>>> - if (!USE_SINGLE_STORE)
>>> + if ( !USE_SINGLE_STORE )
>>> {
>>>
>>> for ( Map.Entry<String, List<Pointer>> entry :
>>> pathTrackingEntries.entrySet() )
>>> @@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
>>> {
>>> if ( !pointer.isFree() )
>>> {
>>> - cube.postBytes( readBytes( pointer ),
>>> PathTrackingEntry.class.getName() );
>>> + CUBE.postBytes( readBytes( pointer ),
>>> PathTrackingEntry.class.getName() );
>>> pointer.freeMemory();
>>> }
>>> }
>>> @@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
>>> {
>>> executorService.shutdownNow();
>>> }
>>> +
>>> +
>>> }
>>>
>>> Modified:
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>> URL:
>>> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>> (original)
>>> +++
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>> Fri Mar 7 06:07:56 2014
>>> @@ -59,7 +59,6 @@ public class PathTracker
>>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
>>> Configuration.getProperty(
>>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
>>> "false" ) );
>>>
>>> -
>>> protected static ExecutorService executorService;
>>>
>>> static
>>>
>>> Modified:
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>> URL:
>>> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>>
>>> ==============================================================================
>>> ---
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>> (original)
>>> +++
>>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>> Fri Mar 7 06:07:56 2014
>>> @@ -63,6 +63,11 @@ public class PathTrackingEntry
>>> */
>>> private int level;
>>>
>>> + public PathTrackingEntry()
>>> + {
>>> + // no op
>>> + }
>>> +
>>> public PathTrackingEntry( String trackingId, String nodeId,
>>> String className, String methodName, long startTime,
>>> long executionTime, int level )
>>> {
>>>
--
Olivier Lamy
Ecetera: http://ecetera.com.au
http://twitter.com/olamy | http://linkedin.com/in/olamy
Re: Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Posted by Romain Manni-Bucau <rm...@gmail.com>.
well if you have figures and say me disruptor is always 10 times
faster than default impl (regarding overall jvm perf and not only this
thread) I'll say: ok keep disruptor only but since i'm not sure it
changes a lot of things i think default shouldnt use disruptor and we
should stay simple and don't bring disruptor transitively
Romain Manni-Bucau
Twitter: @rmannibucau
Blog: http://rmannibucau.wordpress.com/
LinkedIn: http://fr.linkedin.com/in/rmannibucau
Github: https://github.com/rmannibucau
2014-03-07 9:15 GMT+01:00 Jean-Louis MONTEIRO <je...@gmail.com>:
> Romain not sure I got the mandatory with 10.
> BTW big big thx you. Great feature.
> Gonna try to use that and get back to the project.
>
> Jean Louis
> Le 7 mars 2014 08:19, "Romain Manni-Bucau" <rm...@gmail.com> a écrit :
>
>> Hi
>>
>> any issue making disruptor optional and creating two
>> CubePathTrackingDataStore impl (the old one and disruptor one)? Think
>> we shouldn't make disruptor mandatory excepted if perf are > 10.
>>
>> Romain Manni-Bucau
>> Twitter: @rmannibucau
>> Blog: http://rmannibucau.wordpress.com/
>> LinkedIn: http://fr.linkedin.com/in/rmannibucau
>> Github: https://github.com/rmannibucau
>>
>>
>>
>>
>> ---------- Forwarded message ----------
>> From: <ol...@apache.org>
>> Date: 2014-03-07 7:07 GMT+01:00
>> Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
>> agent/store/cube/
>> agent/store/cube/src/main/java/org/apache/sirona/cube/
>> core/src/main/java/org/apache/sirona/tracking/
>> To: commits@sirona.incubator.apache.org
>>
>>
>> Author: olamy
>> Date: Fri Mar 7 06:07:56 2014
>> New Revision: 1575186
>>
>> URL: http://svn.apache.org/r1575186
>> Log:
>> use disruptor to handle amount of messages and not block the
>> application when sending path tracking entries
>>
>> Modified:
>> incubator/sirona/trunk/agent/store/cube/pom.xml
>>
>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>>
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>>
>> Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
>> URL:
>> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
>>
>> ==============================================================================
>> --- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
>> +++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7 06:07:56
>> 2014
>> @@ -32,6 +32,13 @@
>> <dependency>
>> <groupId>org.apache.sirona</groupId>
>> <artifactId>sirona-core</artifactId>
>> + <scope>provided</scope>
>> + </dependency>
>> +
>> + <dependency>
>> + <groupId>com.lmax</groupId>
>> + <artifactId>disruptor</artifactId>
>> + <version>3.2.0</version>
>> </dependency>
>>
>> <dependency>
>> @@ -69,6 +76,34 @@
>> </execution>
>> </executions>
>> </plugin>
>> + <plugin>
>> + <groupId>org.apache.maven.plugins</groupId>
>> + <artifactId>maven-shade-plugin</artifactId>
>> + <executions>
>> + <execution>
>> + <goals>
>> + <goal>shade</goal>
>> + </goals>
>> + <configuration>
>> + <createSourcesJar>true</createSourcesJar>
>> + <shadedArtifactAttached>true</shadedArtifactAttached>
>> + <shadedClassifierName>shaded</shadedClassifierName>
>> + <relocations>
>> + <relocation>
>> + <pattern>com.lmax</pattern>
>> +
>> <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
>> + </relocation>
>> + <!--
>> + <relocation>
>> + <pattern>org.apache.sirona</pattern>
>> + <shadedPattern>org.apache.sirona.sirona</shadedPattern>
>> + </relocation>
>> + -->
>> + </relocations>
>> + </configuration>
>> + </execution>
>> + </executions>
>> + </plugin>
>> </plugins>
>> <pluginManagement>
>> <plugins>
>>
>> Modified:
>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>> URL:
>> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>
>> ==============================================================================
>> ---
>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>> (original)
>> +++
>> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>> Fri Mar 7 06:07:56 2014
>> @@ -17,6 +17,13 @@
>>
>> package org.apache.sirona.cube;
>>
>> +import com.lmax.disruptor.BusySpinWaitStrategy;
>> +import com.lmax.disruptor.EventFactory;
>> +import com.lmax.disruptor.EventHandler;
>> +import com.lmax.disruptor.EventTranslator;
>> +import com.lmax.disruptor.RingBuffer;
>> +import com.lmax.disruptor.dsl.Disruptor;
>> +import com.lmax.disruptor.dsl.ProducerType;
>> import org.apache.sirona.configuration.Configuration;
>> import org.apache.sirona.configuration.ioc.Destroying;
>> import org.apache.sirona.configuration.ioc.IoCs;
>> @@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
>> extends BatchPathTrackingDataStore
>> implements CollectorPathTrackingDataStore
>> {
>> - private final Cube cube = IoCs.findOrCreateInstance(
>> CubeBuilder.class ).build();
>> + private static final Cube CUBE = IoCs.findOrCreateInstance(
>> CubeBuilder.class ).build();
>>
>>
>> private static final boolean USE_EXECUTORS = Boolean.parseBoolean(
>> @@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
>> Configuration.getProperty(
>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
>> "false" ) );
>>
>> + private static final boolean USE_DISRUPTOR = Boolean.parseBoolean(
>> + Configuration.getProperty(
>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.usedisruptor",
>> "true" ) );
>> +
>> protected static ExecutorService executorService;
>>
>> + private static RingBuffer<PathTrackingEntry> RINGBUFFER;
>> +
>> static
>> {
>>
>> @@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
>> executorService = Executors.newFixedThreadPool( threadsNumber
>> );
>>
>> }
>> +
>> + if ( USE_DISRUPTOR )
>> + {
>> + ExecutorService exec = Executors.newCachedThreadPool();
>> +
>> + // FIXME make configurable: ring buffer size and WaitStrategy
>> +
>> + Disruptor<PathTrackingEntry> disruptor =
>> + new Disruptor<PathTrackingEntry>( new
>> EventFactory<PathTrackingEntry>()
>> + {
>> + @Override
>> + public PathTrackingEntry newInstance()
>> + {
>> + return new PathTrackingEntry();
>> + }
>> + }, 2048, exec, ProducerType.SINGLE, new
>> BusySpinWaitStrategy()
>> + );
>> +
>> + final EventHandler<PathTrackingEntry> handler = new
>> EventHandler<PathTrackingEntry>()
>> + {
>> + // event will eventually be recycled by the Disruptor
>> after it wraps
>> + public void onEvent( final PathTrackingEntry entry,
>> final long sequence, final boolean endOfBatch )
>> + throws Exception
>> + {
>> + CUBE.postBytes( SerializeUtils.serialize( entry
>> ), PathTrackingEntry.class.getName() );
>> + }
>> + };
>> +
>> + disruptor.handleEventsWith( handler );
>> +
>> + RINGBUFFER = disruptor.start();
>> + }
>> }
>>
>> @Override
>> - public void store( PathTrackingEntry pathTrackingEntry )
>> + public void store( final PathTrackingEntry pathTrackingEntry )
>> {
>> - cube.postBytes( SerializeUtils.serialize( pathTrackingEntry
>> ), PathTrackingEntry.class.getName() );
>> + if ( USE_DISRUPTOR )
>> + {
>> +
>> + RINGBUFFER.publishEvent( new
>> EventTranslator<PathTrackingEntry>()
>> + {
>> + @Override
>> + public void translateTo( PathTrackingEntry event,
>> long sequence )
>> + {
>> + event.setClassName( pathTrackingEntry.getClassName()
>> );
>> + event.setExecutionTime(
>> pathTrackingEntry.getExecutionTime() );
>> + event.setLevel( pathTrackingEntry.getLevel() );
>> + event.setMethodName(
>> pathTrackingEntry.getMethodName() );
>> + event.setNodeId( pathTrackingEntry.getNodeId() );
>> + event.setStartTime( pathTrackingEntry.getStartTime()
>> );
>> + event.setTrackingId(
>> pathTrackingEntry.getTrackingId() );
>> + }
>> + } );
>> +
>> +
>> + }
>> + else
>> + {
>> + CUBE.postBytes( SerializeUtils.serialize(
>> pathTrackingEntry ), PathTrackingEntry.class.getName() );
>> + }
>> +
>> }
>>
>> @Override
>> protected void pushEntriesByBatch( Map<String, List<Pointer>>
>> pathTrackingEntries )
>> {
>> - if (!USE_SINGLE_STORE)
>> + if ( !USE_SINGLE_STORE )
>> {
>>
>> for ( Map.Entry<String, List<Pointer>> entry :
>> pathTrackingEntries.entrySet() )
>> @@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
>> {
>> if ( !pointer.isFree() )
>> {
>> - cube.postBytes( readBytes( pointer ),
>> PathTrackingEntry.class.getName() );
>> + CUBE.postBytes( readBytes( pointer ),
>> PathTrackingEntry.class.getName() );
>> pointer.freeMemory();
>> }
>> }
>> @@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
>> {
>> executorService.shutdownNow();
>> }
>> +
>> +
>> }
>>
>> Modified:
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>> URL:
>> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>
>> ==============================================================================
>> ---
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>> (original)
>> +++
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>> Fri Mar 7 06:07:56 2014
>> @@ -59,7 +59,6 @@ public class PathTracker
>> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
>> Configuration.getProperty(
>> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
>> "false" ) );
>>
>> -
>> protected static ExecutorService executorService;
>>
>> static
>>
>> Modified:
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>> URL:
>> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>>
>> ==============================================================================
>> ---
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>> (original)
>> +++
>> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>> Fri Mar 7 06:07:56 2014
>> @@ -63,6 +63,11 @@ public class PathTrackingEntry
>> */
>> private int level;
>>
>> + public PathTrackingEntry()
>> + {
>> + // no op
>> + }
>> +
>> public PathTrackingEntry( String trackingId, String nodeId,
>> String className, String methodName, long startTime,
>> long executionTime, int level )
>> {
>>
Re: Fwd: svn commit: r1575186 - in /incubator/sirona/trunk:
agent/store/cube/ agent/store/cube/src/main/java/org/apache/sirona/cube/ core/src/main/java/org/apache/sirona/tracking/
Posted by Jean-Louis MONTEIRO <je...@gmail.com>.
Romain not sure I got the mandatory with 10.
BTW big big thx you. Great feature.
Gonna try to use that and get back to the project.
Jean Louis
Le 7 mars 2014 08:19, "Romain Manni-Bucau" <rm...@gmail.com> a écrit :
> Hi
>
> any issue making disruptor optional and creating two
> CubePathTrackingDataStore impl (the old one and disruptor one)? Think
> we shouldn't make disruptor mandatory excepted if perf are > 10.
>
> Romain Manni-Bucau
> Twitter: @rmannibucau
> Blog: http://rmannibucau.wordpress.com/
> LinkedIn: http://fr.linkedin.com/in/rmannibucau
> Github: https://github.com/rmannibucau
>
>
>
>
> ---------- Forwarded message ----------
> From: <ol...@apache.org>
> Date: 2014-03-07 7:07 GMT+01:00
> Subject: svn commit: r1575186 - in /incubator/sirona/trunk:
> agent/store/cube/
> agent/store/cube/src/main/java/org/apache/sirona/cube/
> core/src/main/java/org/apache/sirona/tracking/
> To: commits@sirona.incubator.apache.org
>
>
> Author: olamy
> Date: Fri Mar 7 06:07:56 2014
> New Revision: 1575186
>
> URL: http://svn.apache.org/r1575186
> Log:
> use disruptor to handle amount of messages and not block the
> application when sending path tracking entries
>
> Modified:
> incubator/sirona/trunk/agent/store/cube/pom.xml
>
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
>
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
>
> Modified: incubator/sirona/trunk/agent/store/cube/pom.xml
> URL:
> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/pom.xml?rev=1575186&r1=1575185&r2=1575186&view=diff
>
> ==============================================================================
> --- incubator/sirona/trunk/agent/store/cube/pom.xml (original)
> +++ incubator/sirona/trunk/agent/store/cube/pom.xml Fri Mar 7 06:07:56
> 2014
> @@ -32,6 +32,13 @@
> <dependency>
> <groupId>org.apache.sirona</groupId>
> <artifactId>sirona-core</artifactId>
> + <scope>provided</scope>
> + </dependency>
> +
> + <dependency>
> + <groupId>com.lmax</groupId>
> + <artifactId>disruptor</artifactId>
> + <version>3.2.0</version>
> </dependency>
>
> <dependency>
> @@ -69,6 +76,34 @@
> </execution>
> </executions>
> </plugin>
> + <plugin>
> + <groupId>org.apache.maven.plugins</groupId>
> + <artifactId>maven-shade-plugin</artifactId>
> + <executions>
> + <execution>
> + <goals>
> + <goal>shade</goal>
> + </goals>
> + <configuration>
> + <createSourcesJar>true</createSourcesJar>
> + <shadedArtifactAttached>true</shadedArtifactAttached>
> + <shadedClassifierName>shaded</shadedClassifierName>
> + <relocations>
> + <relocation>
> + <pattern>com.lmax</pattern>
> +
> <shadedPattern>org.apache.sirona.com.lmax</shadedPattern>
> + </relocation>
> + <!--
> + <relocation>
> + <pattern>org.apache.sirona</pattern>
> + <shadedPattern>org.apache.sirona.sirona</shadedPattern>
> + </relocation>
> + -->
> + </relocations>
> + </configuration>
> + </execution>
> + </executions>
> + </plugin>
> </plugins>
> <pluginManagement>
> <plugins>
>
> Modified:
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> URL:
> http://svn.apache.org/viewvc/incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>
> ==============================================================================
> ---
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> (original)
> +++
> incubator/sirona/trunk/agent/store/cube/src/main/java/org/apache/sirona/cube/CubePathTrackingDataStore.java
> Fri Mar 7 06:07:56 2014
> @@ -17,6 +17,13 @@
>
> package org.apache.sirona.cube;
>
> +import com.lmax.disruptor.BusySpinWaitStrategy;
> +import com.lmax.disruptor.EventFactory;
> +import com.lmax.disruptor.EventHandler;
> +import com.lmax.disruptor.EventTranslator;
> +import com.lmax.disruptor.RingBuffer;
> +import com.lmax.disruptor.dsl.Disruptor;
> +import com.lmax.disruptor.dsl.ProducerType;
> import org.apache.sirona.configuration.Configuration;
> import org.apache.sirona.configuration.ioc.Destroying;
> import org.apache.sirona.configuration.ioc.IoCs;
> @@ -37,7 +44,7 @@ public class CubePathTrackingDataStore
> extends BatchPathTrackingDataStore
> implements CollectorPathTrackingDataStore
> {
> - private final Cube cube = IoCs.findOrCreateInstance(
> CubeBuilder.class ).build();
> + private static final Cube CUBE = IoCs.findOrCreateInstance(
> CubeBuilder.class ).build();
>
>
> private static final boolean USE_EXECUTORS = Boolean.parseBoolean(
> @@ -46,8 +53,13 @@ public class CubePathTrackingDataStore
> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
> Configuration.getProperty(
> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
> "false" ) );
>
> + private static final boolean USE_DISRUPTOR = Boolean.parseBoolean(
> + Configuration.getProperty(
> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.usedisruptor",
> "true" ) );
> +
> protected static ExecutorService executorService;
>
> + private static RingBuffer<PathTrackingEntry> RINGBUFFER;
> +
> static
> {
>
> @@ -58,18 +70,74 @@ public class CubePathTrackingDataStore
> executorService = Executors.newFixedThreadPool( threadsNumber
> );
>
> }
> +
> + if ( USE_DISRUPTOR )
> + {
> + ExecutorService exec = Executors.newCachedThreadPool();
> +
> + // FIXME make configurable: ring buffer size and WaitStrategy
> +
> + Disruptor<PathTrackingEntry> disruptor =
> + new Disruptor<PathTrackingEntry>( new
> EventFactory<PathTrackingEntry>()
> + {
> + @Override
> + public PathTrackingEntry newInstance()
> + {
> + return new PathTrackingEntry();
> + }
> + }, 2048, exec, ProducerType.SINGLE, new
> BusySpinWaitStrategy()
> + );
> +
> + final EventHandler<PathTrackingEntry> handler = new
> EventHandler<PathTrackingEntry>()
> + {
> + // event will eventually be recycled by the Disruptor
> after it wraps
> + public void onEvent( final PathTrackingEntry entry,
> final long sequence, final boolean endOfBatch )
> + throws Exception
> + {
> + CUBE.postBytes( SerializeUtils.serialize( entry
> ), PathTrackingEntry.class.getName() );
> + }
> + };
> +
> + disruptor.handleEventsWith( handler );
> +
> + RINGBUFFER = disruptor.start();
> + }
> }
>
> @Override
> - public void store( PathTrackingEntry pathTrackingEntry )
> + public void store( final PathTrackingEntry pathTrackingEntry )
> {
> - cube.postBytes( SerializeUtils.serialize( pathTrackingEntry
> ), PathTrackingEntry.class.getName() );
> + if ( USE_DISRUPTOR )
> + {
> +
> + RINGBUFFER.publishEvent( new
> EventTranslator<PathTrackingEntry>()
> + {
> + @Override
> + public void translateTo( PathTrackingEntry event,
> long sequence )
> + {
> + event.setClassName( pathTrackingEntry.getClassName()
> );
> + event.setExecutionTime(
> pathTrackingEntry.getExecutionTime() );
> + event.setLevel( pathTrackingEntry.getLevel() );
> + event.setMethodName(
> pathTrackingEntry.getMethodName() );
> + event.setNodeId( pathTrackingEntry.getNodeId() );
> + event.setStartTime( pathTrackingEntry.getStartTime()
> );
> + event.setTrackingId(
> pathTrackingEntry.getTrackingId() );
> + }
> + } );
> +
> +
> + }
> + else
> + {
> + CUBE.postBytes( SerializeUtils.serialize(
> pathTrackingEntry ), PathTrackingEntry.class.getName() );
> + }
> +
> }
>
> @Override
> protected void pushEntriesByBatch( Map<String, List<Pointer>>
> pathTrackingEntries )
> {
> - if (!USE_SINGLE_STORE)
> + if ( !USE_SINGLE_STORE )
> {
>
> for ( Map.Entry<String, List<Pointer>> entry :
> pathTrackingEntries.entrySet() )
> @@ -78,7 +146,7 @@ public class CubePathTrackingDataStore
> {
> if ( !pointer.isFree() )
> {
> - cube.postBytes( readBytes( pointer ),
> PathTrackingEntry.class.getName() );
> + CUBE.postBytes( readBytes( pointer ),
> PathTrackingEntry.class.getName() );
> pointer.freeMemory();
> }
> }
> @@ -91,4 +159,6 @@ public class CubePathTrackingDataStore
> {
> executorService.shutdownNow();
> }
> +
> +
> }
>
> Modified:
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> URL:
> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>
> ==============================================================================
> ---
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> (original)
> +++
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTracker.java
> Fri Mar 7 06:07:56 2014
> @@ -59,7 +59,6 @@ public class PathTracker
> private static boolean USE_SINGLE_STORE = Boolean.parseBoolean(
> Configuration.getProperty(
> Configuration.CONFIG_PROPERTY_PREFIX + "pathtracking.singlestore",
> "false" ) );
>
> -
> protected static ExecutorService executorService;
>
> static
>
> Modified:
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> URL:
> http://svn.apache.org/viewvc/incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java?rev=1575186&r1=1575185&r2=1575186&view=diff
>
> ==============================================================================
> ---
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> (original)
> +++
> incubator/sirona/trunk/core/src/main/java/org/apache/sirona/tracking/PathTrackingEntry.java
> Fri Mar 7 06:07:56 2014
> @@ -63,6 +63,11 @@ public class PathTrackingEntry
> */
> private int level;
>
> + public PathTrackingEntry()
> + {
> + // no op
> + }
> +
> public PathTrackingEntry( String trackingId, String nodeId,
> String className, String methodName, long startTime,
> long executionTime, int level )
> {
>