You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/07/22 17:55:56 UTC
[1/2] incubator-usergrid git commit: add serialization timers
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-dev 89dd0ad98 -> e85753ca6
add serialization timers
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cf141e63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cf141e63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cf141e63
Branch: refs/heads/two-dot-o-dev
Commit: cf141e63fd42cc9f71483bc334fb84bdfae001e4
Parents: b1393b4
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Jul 22 09:47:31 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Jul 22 09:47:31 2015 -0600
----------------------------------------------------------------------
.../MvccEntitySerializationStrategyV2Impl.java | 21 +++++++--
.../MvccEntitySerializationStrategyV3Impl.java | 47 ++++++++++++--------
2 files changed, 47 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf141e63/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
index e81baab..26ec5ec 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -23,6 +23,8 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
import java.nio.ByteBuffer;
import java.util.UUID;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Timer;
import com.fasterxml.jackson.core.Version;
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.apache.usergrid.persistence.collection.MvccEntity;
@@ -37,6 +39,7 @@ import org.apache.usergrid.persistence.core.astyanax.FieldBufferSerializer;
import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -78,9 +81,9 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
@Inject
- public MvccEntitySerializationStrategyV2Impl( final Keyspace keyspace, final SerializationFig serializationFig, final CassandraFig cassandraFig ) {
+ public MvccEntitySerializationStrategyV2Impl( final Keyspace keyspace, final SerializationFig serializationFig, final CassandraFig cassandraFig, final MetricsFactory metricsFactory ) {
super( keyspace, serializationFig, cassandraFig );
- entitySerializer = new EntitySerializer( serializationFig );
+ entitySerializer = new EntitySerializer( serializationFig, metricsFactory );
}
@@ -111,6 +114,9 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
private final SmileFactory SMILE_FACTORY = new SmileFactory();
private final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+ private final Histogram bytesInHistorgram;
+ private final Histogram bytesOutHistorgram;
+ private final Timer bytesOutTimer;
private SerializationFig serializationFig;
@@ -123,13 +129,17 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
private byte VERSION = 1;
- public EntitySerializer( final SerializationFig serializationFig) {
+ public EntitySerializer( final SerializationFig serializationFig, final MetricsFactory metricsFactory) {
this.serializationFig = serializationFig;
// SimpleModule listModule = new SimpleModule("ListFieldModule", new Version(1, 0, 0, null,null,null))
// .addAbstractTypeMapping(ListField.class, ArrayField.class);
// MAPPER.registerModule(listModule);
// causes slowness
MAPPER.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
+ this.bytesOutHistorgram = metricsFactory.getHistogram(MvccEntitySerializationStrategyV2Impl.class, "bytes.out");
+ this.bytesInHistorgram = metricsFactory.getHistogram(MvccEntitySerializationStrategyV2Impl.class, "bytes.in");
+ this.bytesOutTimer = metricsFactory.getTimer(MvccEntitySerializationStrategyV2Impl.class, "bytes.out");
+
}
@@ -185,6 +195,8 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
builder.addBytes( entityBytes );
+ bytesInHistorgram.update(entityBytes.length);
+
return FIELD_BUFFER_SERIALIZER.toByteBuffer( builder.build() );
}
@@ -231,7 +243,10 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
byte[] array = parser.readBytes();
try {
+ Timer.Context time = bytesOutTimer.time();
+ bytesOutHistorgram.update(array == null ? 0 : array.length);
storedEntity = MAPPER.readValue( array, Entity.class );
+ time.stop();
}
catch ( Exception e ) {
throw new DataCorruptionException( "Unable to read entity data", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf141e63/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index 8be640d..81a0e6d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -9,11 +9,14 @@ import java.util.Iterator;
import java.util.List;
import java.util.UUID;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Timer;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.netflix.astyanax.serializers.StringSerializer;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,11 +94,11 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
@Inject
public MvccEntitySerializationStrategyV3Impl( final Keyspace keyspace, final SerializationFig serializationFig,
- final CassandraFig cassandraFig ) {
+ final CassandraFig cassandraFig, final MetricsFactory metricsFactory ) {
this.keyspace = keyspace;
this.serializationFig = serializationFig;
this.cassandraFig = cassandraFig;
- this.entitySerializer = new EntitySerializer( serializationFig );
+ this.entitySerializer = new EntitySerializer( serializationFig, metricsFactory );
}
@@ -368,13 +371,19 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
private final JsonFactory JSON_FACTORY = new JsonFactory();
private final ObjectMapper MAPPER = new ObjectMapper( JSON_FACTORY );
+ private final Histogram bytesInHistorgram;
+ private final Histogram bytesOutHistorgram;
+ private final Timer bytesOutTimer;
private SerializationFig serializationFig;
- public EntitySerializer( final SerializationFig serializationFig ) {
+ public EntitySerializer( final SerializationFig serializationFig, final MetricsFactory metricsFactory) {
this.serializationFig = serializationFig;
+ this.bytesOutHistorgram = metricsFactory.getHistogram(MvccEntitySerializationStrategyV3Impl.class, "bytes.out");
+ this.bytesOutTimer = metricsFactory.getTimer(MvccEntitySerializationStrategyV3Impl.class, "bytes.out");
+ this.bytesInHistorgram = metricsFactory.getHistogram(MvccEntitySerializationStrategyV3Impl.class, "bytes.in");
// mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
// causes slowness
@@ -384,42 +393,41 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
@Override
public ByteBuffer toByteBuffer( final EntityWrapper wrapper ) {
- if ( wrapper == null ) {
+ if (wrapper == null) {
return null;
}
-
+ final byte[] wrapperBytes;
//mark this version as empty
- if ( wrapper.getEntityMap() == null ) {
+ if (wrapper.getEntityMap() == null) {
//we're empty
try {
return ByteBuffer.wrap(MAPPER.writeValueAsBytes(wrapper));
- }catch (JsonProcessingException jpe){
- throw new RuntimeException( "Unable to serialize entity", jpe );
+ } catch (JsonProcessingException jpe) {
+ throw new RuntimeException("Unable to serialize entity", jpe);
}
}
//we have an entity but status is not complete don't allow it
- if ( wrapper.getStatus() != MvccEntity.Status.COMPLETE ) {
- throw new UnsupportedOperationException( "Only states " + MvccEntity.Status.DELETED + " and " + MvccEntity.Status.COMPLETE + " are supported" );
+ if (wrapper.getStatus() != MvccEntity.Status.COMPLETE) {
+ throw new UnsupportedOperationException("Only states " + MvccEntity.Status.DELETED + " and " + MvccEntity.Status.COMPLETE + " are supported");
}
wrapper.setStatus(MvccEntity.Status.COMPLETE);
//Convert to internal entity map
- final byte[] wrapperBytes;
try {
wrapperBytes = MAPPER.writeValueAsBytes(wrapper);
final int maxEntrySize = serializationFig.getMaxEntitySize();
+ bytesInHistorgram.update(wrapperBytes.length);
if (wrapperBytes.length > maxEntrySize) {
throw new EntityTooLargeException(Entity.fromMap(wrapper.getEntityMap()), maxEntrySize, wrapperBytes.length,
"Your entity cannot exceed " + maxEntrySize + " bytes. The entity you tried to save was "
+ wrapperBytes.length + " bytes");
}
- }
- catch ( JsonProcessingException jpe ) {
- throw new RuntimeException( "Unable to serialize entity", jpe );
+ } catch (JsonProcessingException jpe) {
+ throw new RuntimeException("Unable to serialize entity", jpe);
}
return ByteBuffer.wrap(wrapperBytes);
@@ -441,14 +449,17 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
try {
- entityWrapper = MAPPER.readValue(byteBuffer.array(), EntityWrapper.class);
-
+ Timer.Context time = bytesOutTimer.time();
+ byte[] arr = byteBuffer.array();
+ bytesOutHistorgram.update( arr == null ? 0 : arr.length);
+ entityWrapper = MAPPER.readValue(arr, EntityWrapper.class);
+ time.stop();
}
catch ( Exception e ) {
- if( log.isDebugEnabled() ){
+ if (log.isDebugEnabled()) {
log.debug("Entity Wrapper Deserialized: " + StringSerializer.get().fromByteBuffer(byteBuffer));
}
- throw new DataCorruptionException( "Unable to read entity data", e );
+ throw new DataCorruptionException("Unable to read entity data", e);
}
// it's been deleted, remove it
[2/2] incubator-usergrid git commit: Merge branch 'two-dot-o-dev' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-dev
Posted by sf...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e85753ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e85753ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e85753ca
Branch: refs/heads/two-dot-o-dev
Commit: e85753ca6ff24282d955921247f81e40d1bb9869
Parents: cf141e6 89dd0ad
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Jul 22 09:55:48 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Jul 22 09:55:48 2015 -0600
----------------------------------------------------------------------
stack/pom.xml | 2 +-
stack/tools/pom.xml | 6 +
.../org/apache/usergrid/tools/ExportAdmins.java | 117 ++--
.../org/apache/usergrid/tools/ExportApp.java | 536 +++++++++++++++++++
.../usergrid/tools/ExportDataCreator.java | 244 +++++++--
.../usergrid/tools/ExportingToolBase.java | 2 +-
.../org/apache/usergrid/tools/ImportAdmins.java | 226 ++++++--
stack/tools/src/main/resources/log4j.properties | 5 +
.../apache/usergrid/tools/ExportAppTest.java | 118 ++++
.../usergrid/tools/ExportImportAdminsTest.java | 71 ++-
...adata.usergrid-management.1433331614293.json | 52 ++
...users.usergrid-management.1433331614293.json | 12 +
12 files changed, 1224 insertions(+), 167 deletions(-)
----------------------------------------------------------------------