You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/08/20 22:01:37 UTC
incubator-usergrid git commit: add entity size
Repository: incubator-usergrid
Updated Branches:
refs/heads/entity-size [created] 5e56f1b9c
add entity size
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5e56f1b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5e56f1b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5e56f1b9
Branch: refs/heads/entity-size
Commit: 5e56f1b9cdec740c9907350ed1b585f09f06752a
Parents: 72016ea
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Aug 20 12:59:22 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Aug 20 12:59:22 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 5 ++--
.../usergrid/persistence/AbstractEntity.java | 7 ++++++
.../org/apache/usergrid/persistence/Entity.java | 3 +++
.../persistence/collection/MvccEntity.java | 6 +++++
.../mvcc/entity/impl/MvccEntityImpl.java | 25 +++++++++++++++++---
.../collection/mvcc/stage/delete/MarkStart.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 4 ++--
.../MvccEntitySerializationStrategyV3Impl.java | 24 +++++++++++++------
...ccEntitySerializationStrategyV3ImplTest.java | 1 -
.../persistence/model/entity/Entity.java | 9 +++++++
.../index/impl/EntityToMapConverter.java | 5 +++-
.../persistence/index/impl/IndexingUtils.java | 2 ++
12 files changed, 76 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 6611b6f..92464f8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2615,7 +2615,8 @@ public class CpEntityManager implements EntityManager {
} );
}
- cpEntity = ecm .write( cpEntity ).toBlocking().last();
+ cpEntity = ecm.write( cpEntity ).toBlocking().last();
+ entity.setSize(cpEntity.getSize());
if(logger.isDebugEnabled()) {
logger.debug( "Wrote {}:{} version {}", new Object[] {
@@ -2647,7 +2648,7 @@ public class CpEntityManager implements EntityManager {
String collectionName = Schema.defaultCollectionName( eType );
CpRelationManager cpr = ( CpRelationManager ) getRelationManager( getApplication() );
- cpr.addToCollection( collectionName, entity);
+ cpr.addToCollection( collectionName, entity );
// Invoke counters
incrementEntityCollection( collectionName, timestamp );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java b/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java
index afe246d..59106ba 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/AbstractEntity.java
@@ -62,6 +62,7 @@ public abstract class AbstractEntity implements Entity {
protected Map<String, Object> dynamic_properties = new TreeMap<String, Object>( String.CASE_INSENSITIVE_ORDER );
protected Map<String, Set<Object>> dynamic_sets = new TreeMap<String, Set<Object>>( String.CASE_INSENSITIVE_ORDER );
+ protected long size;
@Override
@@ -96,7 +97,12 @@ public abstract class AbstractEntity implements Entity {
return new SimpleId( uuid, getType() );
}
+ @Override
+ public void setSize(final long size){this.size = size;}
+
+ @Override
+ public long getSize(){return size;}
@Override
@EntityProperty(indexed = true, required = true, mutable = false)
@@ -363,4 +369,5 @@ public abstract class AbstractEntity implements Entity {
}
return true;
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java
index 2030bd1..b6176b8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Entity.java
@@ -102,4 +102,7 @@ public interface Entity extends EntityRef, Comparable<Entity> {
@JsonAnyGetter
public abstract Map<String, Object> getDynamicProperties();
+
+ long getSize();
+ void setSize(long size);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java
index c89838e..53142c8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java
@@ -68,4 +68,10 @@ public interface MvccEntity extends EntityVersion{
* Get the status of the entity
*/
Status getStatus();
+
+ /**
+ * entity byte size
+ * @return
+ */
+ long getSize();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java
index 14889bd..e0066ba 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java
@@ -38,24 +38,34 @@ public class MvccEntityImpl implements MvccEntity {
private final UUID version;
private final Optional<Entity> entity;
private final Status status;
+ private final long size;
public MvccEntityImpl( final Id entityId, final UUID version, final Status status, final Entity entity ) {
- this( entityId, version, status, Optional.of( entity ) );
+ this( entityId, version, status, entity, 0 );
}
-
+ public MvccEntityImpl( final Id entityId, final UUID version, final Status status, final Entity entity, final long size ) {
+ this( entityId, version, status, Optional.of( entity ), size);
+ }
public MvccEntityImpl(
- final Id entityId, final UUID version, final Status status, final Optional<Entity> entity ) {
+ final Id entityId, final UUID version, final Status status, final Optional<Entity> entity ) {
+ this( entityId, version, status, entity , 0);
+ }
+
+ public MvccEntityImpl(
+ final Id entityId, final UUID version, final Status status, final Optional<Entity> entity, final long size ) {
Preconditions.checkNotNull( entityId, "entity id is required" );
Preconditions.checkNotNull( version, "version id is required" );
Preconditions.checkNotNull( status, "status is required" );
Preconditions.checkNotNull( entity, "entity is required" );
+ Preconditions.checkNotNull( size, "size is required" );
this.entityId = entityId;
this.version = version;
this.entity = entity;
this.status = status;
+ this.size = size;
}
@@ -82,6 +92,11 @@ public class MvccEntityImpl implements MvccEntity {
return status;
}
+ @Override
+ public long getSize() {
+ return size;
+ }
+
@Override
public boolean equals( final Object o ) {
@@ -104,6 +119,10 @@ public class MvccEntityImpl implements MvccEntity {
return false;
}
+ if( size != that.size){
+ return false;
+ }
+
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java
index 040e893..3c4ea95 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java
@@ -106,7 +106,7 @@ public class MarkStart implements Func1<CollectionIoEvent<Id>, CollectionIoEvent
//create the mvcc entity for the next stage
final MvccEntityImpl nextStage = new MvccEntityImpl(
- entityId, version, MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
+ entityId, version, MvccEntity.Status.COMPLETE, Optional.<Entity>absent(), 0 );
return new CollectionIoEvent<MvccEntity>( applicationScope, nextStage );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index 6fa539a..457664f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -424,7 +424,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
+ " write was truncated.", id, version, e );
//return an empty entity, we can never load this one, and we don't want it to bring the system
//to a grinding halt
- return new MvccEntityImpl( id, version, MvccEntity.Status.DELETED, Optional.<Entity>absent() );
+ return new MvccEntityImpl( id, version, MvccEntity.Status.DELETED, Optional.<Entity>absent(),0 );
}
//Inject the id into it.
@@ -432,7 +432,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
EntityUtils.setId( deSerialized.entity.get(), id );
}
- return new MvccEntityImpl( id, version, deSerialized.status, deSerialized.entity );
+ return new MvccEntityImpl( id, version, deSerialized.status, deSerialized.entity, 0 );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
index 81a0e6d..2f4c625 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -112,7 +112,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
Optional<EntityMap> map = EntityMap.fromEntity(entity.getEntity());
return doWrite( applicationScope, entityId, version, colMutation -> colMutation.putColumn( COL_VALUE,
- entitySerializer.toByteBuffer( new EntityWrapper(entityId,entity.getVersion(), entity.getStatus(), map.isPresent() ? map.get() : null ) ) ) );
+ entitySerializer.toByteBuffer( new EntityWrapper(entityId,entity.getVersion(), entity.getStatus(), map.isPresent() ? map.get() : null, 0 ) ) ) );
}
@@ -268,7 +268,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
return doWrite(applicationScope, entityId, version, colMutation ->
colMutation.putColumn(COL_VALUE,
- entitySerializer.toByteBuffer(new EntityWrapper(entityId, version, MvccEntity.Status.DELETED, null))
+ entitySerializer.toByteBuffer(new EntityWrapper(entityId, version, MvccEntity.Status.DELETED, null, 0))
)
);
}
@@ -355,10 +355,10 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
//return an empty entity, we can never load this one, and we don't want it to bring the system
//to a grinding halt
//TODO fix this
- return new MvccEntityImpl( id, UUIDGenerator.newTimeUUID(), MvccEntity.Status.DELETED, Optional.<Entity>absent() );
+ return new MvccEntityImpl( id, UUIDGenerator.newTimeUUID(), MvccEntity.Status.DELETED, Optional.<Entity>absent(),0 );
}
Optional<Entity> entity = deSerialized.getOptionalEntity() ;
- return new MvccEntityImpl( id, deSerialized.getVersion(), deSerialized.getStatus(), entity );
+ return new MvccEntityImpl( id, deSerialized.getVersion(), deSerialized.getStatus(), entity, deSerialized.getSize());
}
}
@@ -453,6 +453,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
byte[] arr = byteBuffer.array();
bytesOutHistorgram.update( arr == null ? 0 : arr.length);
entityWrapper = MAPPER.readValue(arr, EntityWrapper.class);
+ entityWrapper.size = arr.length;
time.stop();
}
catch ( Exception e ) {
@@ -464,7 +465,7 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
// it's been deleted, remove it
if ( entityWrapper.getEntityMap() == null) {
- return new EntityWrapper( entityWrapper.getId(), entityWrapper.getVersion(),MvccEntity.Status.DELETED,null );
+ return new EntityWrapper( entityWrapper.getId(), entityWrapper.getVersion(),MvccEntity.Status.DELETED,null,0 );
}
entityWrapper.setStatus(MvccEntity.Status.COMPLETE);
@@ -482,15 +483,17 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
private MvccEntity.Status status;
private UUID version;
private EntityMap entityMap;
+ private long size;
public EntityWrapper( ) {
}
- public EntityWrapper( final Id id , final UUID version, final MvccEntity.Status status, final EntityMap entity ) {
+ public EntityWrapper( final Id id , final UUID version, final MvccEntity.Status status, final EntityMap entity, final long size ) {
this.setStatus(status);
this.version= version;
this.entityMap = entity;
this.id = id;
+ this.size = size;
}
/**
@@ -525,7 +528,9 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
@JsonIgnore
public Optional<Entity> getOptionalEntity() {
- Optional<Entity> entityReturn = Optional.fromNullable(Entity.fromMap(getEntityMap()));
+ Entity entity = Entity.fromMap(getEntityMap());
+ entity.setSize(getSize());
+ Optional<Entity> entityReturn = Optional.fromNullable(entity);
//Inject the id into it.
if (entityReturn.isPresent()) {
EntityUtils.setId(entityReturn.get(), getId());
@@ -534,6 +539,11 @@ public class MvccEntitySerializationStrategyV3Impl implements MvccEntitySerializ
;
return entityReturn;
}
+
+ @JsonIgnore
+ public long getSize() {
+ return size;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
index 1df22f6..d55894b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
@@ -54,7 +54,6 @@ public class MvccEntitySerializationStrategyV3ImplTest extends MvccEntitySeriali
@Test( expected = UnsupportedOperationException.class )
public void loadDescendingHistory() throws ConnectionException {
-
final String name = "test";
final Id applicationId = new SimpleId( "application" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
index cd75544..013b37a 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
@@ -59,6 +59,9 @@ public class Entity extends EntityObject {
@JsonProperty
private UUID version;
+ @JsonIgnore
+ private long size;
+
/**
* Create an entity with the given type and id. Should be used for all update operations to an existing entity
@@ -155,4 +158,10 @@ public class Entity extends EntityObject {
return getVersion() != null;
}
+ public long getSize() {
+ return this.size;
+ }
+ public void setSize(long size) {
+ this.size = size;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
index 567068c..1b0d110 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -35,6 +35,7 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_SEAR
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.EDGE_TIMESTAMP_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_FIELDS;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_SIZE_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_TYPE_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.applicationId;
@@ -71,7 +72,7 @@ public class EntityToMapConverter {
outputEntity.put( ENTITY_VERSION_FIELDNAME, entity.getVersion() );
- outputEntity.put( ENTITY_TYPE_FIELDNAME, getType( applicationScope, entityId ) );
+ outputEntity.put( ENTITY_TYPE_FIELDNAME, getType( applicationScope, entityId));
outputEntity.put( APPLICATION_ID_FIELDNAME, applicationId( applicationScope.getApplication() ) );
@@ -83,6 +84,8 @@ public class EntityToMapConverter {
outputEntity.put( EDGE_TIMESTAMP_FIELDNAME, indexEdge.getTimestamp() );
+ outputEntity.put( ENTITY_SIZE_FIELDNAME, entity.getSize() );
+
//add the context for filtering later
outputEntity.put( EDGE_SEARCH_FIELDNAME, IndexingUtils.createContextName( applicationScope, indexEdge ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e56f1b9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index e82a082..9e06fa6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -84,6 +84,8 @@ public class IndexingUtils {
public static final String ENTITY_ID_FIELDNAME = "entityId";
+ public static final String ENTITY_SIZE_FIELDNAME = "entitySize";
+
public static final String ENTITY_VERSION_FIELDNAME = "entityVersion";
public static final String ENTITY_TYPE_FIELDNAME = "entityType";