You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/01/29 21:34:19 UTC

incubator-usergrid git commit: added new entity object for serialization

Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-365 198f4795a -> 481ea5e9b


added new entity object for serialization


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/481ea5e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/481ea5e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/481ea5e9

Branch: refs/heads/USERGRID-365
Commit: 481ea5e9bd26f082f634f04032f2836278e58684
Parents: 198f479
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Jan 29 11:46:01 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Jan 29 11:46:01 2015 -0700

----------------------------------------------------------------------
 .../MvccEntitySerializationStrategyV3Impl.java  | 212 +++++++++++++++++++
 .../serialization/impl/SerializationModule.java |   3 +
 ...ccEntitySerializationStrategyV3ImplTest.java |  33 +++
 .../MvccEntitySerializationStrategyV3Test.java  | 108 ++++++++++
 .../persistence/core/guice/FutureImpl.java      |  19 ++
 .../persistence/model/entity/Entity.java        |  18 +-
 .../persistence/model/entity/EntityMap.java     |  24 +++
 .../model/entity/EntityToMapConverter.java      | 125 +++++++++++
 .../model/entity/MapToEntityConverter.java      | 195 +++++++++++++++++
 .../persistence/model/field/ListField.java      |   2 +-
 .../index/impl/EntityIndexMapUtils.java         |  52 ++---
 11 files changed, 763 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
new file mode 100644
index 0000000..ce8dbef
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -0,0 +1,212 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+import com.netflix.astyanax.serializers.UUIDSerializer;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
+import org.apache.usergrid.persistence.collection.exception.EntityTooLargeException;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.*;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.EntityMap;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.UUID;
+
+/**
+ * Classy class class.
+ */
+public class MvccEntitySerializationStrategyV3Impl extends MvccEntitySerializationStrategyImpl {
+    private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+    private static final CollectionScopedRowKeySerializer<Id> ROW_KEY_SER =
+            new CollectionScopedRowKeySerializer<>(ID_SER);
+
+
+    private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> CF_ENTITY_DATA =
+            new MultiTennantColumnFamily<>("Entity_Version_Data_V2", ROW_KEY_SER, UUIDSerializer.get());
+
+    private static final FieldBufferSerializer FIELD_BUFFER_SERIALIZER = FieldBufferSerializer.get();
+
+
+    private final EntitySerializer entitySerializer;
+
+
+    @Inject
+    public MvccEntitySerializationStrategyV3Impl(final Keyspace keyspace, final SerializationFig serializationFig, final CassandraFig cassandraFig) {
+        super(keyspace, serializationFig, cassandraFig);
+        entitySerializer = new EntitySerializer(serializationFig);
+    }
+
+
+    @Override
+    protected AbstractSerializer<MvccEntitySerializationStrategyImpl.EntityWrapper> getEntitySerializer() {
+        return entitySerializer;
+    }
+
+
+    @Override
+    protected MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> getColumnFamily() {
+        return CF_ENTITY_DATA;
+    }
+
+
+    /**
+     * We should only ever create this once, since this impl is a singleton
+     */
+    public final class EntitySerializer extends AbstractSerializer<MvccEntitySerializationStrategyImpl.EntityWrapper> {
+
+
+        private final SmileFactory SMILE_FACTORY = new SmileFactory();
+
+        private final ObjectMapper MAPPER = new ObjectMapper(SMILE_FACTORY);
+
+
+        private SerializationFig serializationFig;
+
+
+        private byte STATE_COMPLETE = 0;
+        private byte STATE_DELETED = 1;
+        private byte STATE_PARTIAL = 2;
+
+        private byte VERSION = 1;
+
+
+        public EntitySerializer(final SerializationFig serializationFig) {
+            this.serializationFig = serializationFig;
+
+            //                mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
+            // causes slowness
+            MAPPER.enableDefaultTypingAsProperty(ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class");
+        }
+
+
+        @Override
+        public ByteBuffer toByteBuffer(final MvccEntitySerializationStrategyImpl.EntityWrapper wrapper) {
+            if (wrapper == null) {
+                return null;
+            }
+
+            //we always have a max of 3 fields
+            FieldBufferBuilder builder = new FieldBufferBuilder(3);
+
+            builder.addByte(VERSION);
+
+            //mark this version as empty
+            if (!wrapper.entity.isPresent()) {
+                //we're empty
+                builder.addByte(STATE_DELETED);
+
+
+                return FIELD_BUFFER_SERIALIZER.toByteBuffer(builder.build());
+            }
+
+            //we have an entity
+
+            if (wrapper.status == MvccEntity.Status.COMPLETE) {
+                builder.addByte(STATE_COMPLETE);
+            } else {
+                builder.addByte(STATE_PARTIAL);
+            }
+
+
+            //Get Entity
+            final Entity entity = wrapper.entity.get();
+            //Convert to internal entity map
+            final EntityMap entityMap = new EntityMap(entity);
+            final byte[] entityBytes;
+            try {
+                entityBytes = MAPPER.writeValueAsBytes(entityMap);
+            } catch (Exception e) {
+                throw new RuntimeException("Unable to serialize entity", e);
+            }
+
+
+            final int maxEntrySize = serializationFig.getMaxEntitySize();
+
+            if (entityBytes.length > maxEntrySize) {
+                throw new EntityTooLargeException(entity, maxEntrySize, entityBytes.length,
+                        "Your entity cannot exceed " + maxEntrySize + " bytes. The entity you tried to save was "
+                                + entityBytes.length + " bytes");
+            }
+
+            builder.addBytes(entityBytes);
+
+            return FIELD_BUFFER_SERIALIZER.toByteBuffer(builder.build());
+        }
+
+
+        @Override
+        public MvccEntitySerializationStrategyImpl.EntityWrapper fromByteBuffer(final ByteBuffer byteBuffer) {
+
+            /**
+             * We intentionally turn data corruption exceptions when we're unable to de-serialize
+             * the data in cassandra.  If this occurs, we'll never be able to de-serialize it
+             * and it should be considered lost.  This is an error that is occurring due to a bug
+             * in serializing the entity.  This is a lazy recognition + repair signal for deployment with
+             * existing systems.
+             */
+
+            final FieldBuffer fieldBuffer;
+
+            try {
+                fieldBuffer = FIELD_BUFFER_SERIALIZER.fromByteBuffer(byteBuffer);
+            } catch (Exception e) {
+                throw new DataCorruptionException("Unable to de-serialze entity", e);
+            }
+
+            FieldBufferParser parser = new FieldBufferParser(fieldBuffer);
+
+
+            byte version = parser.readByte();
+
+            if (VERSION != version) {
+                throw new UnsupportedOperationException("A version of type " + version + " is unsupported");
+            }
+
+            byte state = parser.readByte();
+
+            // it's been deleted, remove it
+
+            if (STATE_DELETED == state) {
+                return new MvccEntitySerializationStrategyImpl.EntityWrapper(MvccEntity.Status.COMPLETE, Optional.<Entity>absent());
+            }
+
+            EntityMap storedEntity;
+
+            byte[] array = parser.readBytes();
+
+            try {
+                storedEntity = MAPPER.readValue(array, EntityMap.class);
+            } catch (Exception e) {
+                throw new DataCorruptionException("Unable to read entity data", e);
+            }
+
+            Entity entityObject = new Entity(storedEntity);
+
+            final Optional<Entity> entity = Optional.of(entityObject);
+
+            if (STATE_COMPLETE == state) {
+                return new MvccEntitySerializationStrategyImpl.EntityWrapper(MvccEntity.Status.COMPLETE, entity);
+            }
+
+            // it's partial by default
+            return new MvccEntitySerializationStrategyImpl.EntityWrapper(MvccEntity.Status.PARTIAL, entity);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index 33a092d..ac2b48c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -22,6 +22,7 @@ import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationSt
 import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
 import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
 import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.FutureImpl;
 import org.apache.usergrid.persistence.core.guice.PreviousImpl;
 import org.apache.usergrid.persistence.core.guice.ProxyImpl;
 import org.apache.usergrid.persistence.core.migration.schema.Migration;
@@ -47,6 +48,8 @@ public class SerializationModule extends AbstractModule {
                                                      .to( MvccEntitySerializationStrategyV1Impl.class );
         bind( MvccEntitySerializationStrategy.class ).annotatedWith( CurrentImpl.class )
                                                      .to( MvccEntitySerializationStrategyV2Impl.class );
+        bind( MvccEntitySerializationStrategy.class ).annotatedWith( FutureImpl.class )
+                .to(MvccEntitySerializationStrategyV3Impl.class);
         bind( MvccEntitySerializationStrategy.class ).annotatedWith( ProxyImpl.class )
                                                      .to( MvccEntitySerializationStrategyProxyImpl.class );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
new file mode 100644
index 0000000..4a3d761
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
@@ -0,0 +1,33 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
+import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.FutureImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.junit.runner.RunWith;
+
+/**
+ * Classy class class.
+ */
+
+
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class MvccEntitySerializationStrategyV3ImplTest extends MvccEntitySerializationStrategyV3Test {
+    @Inject
+    @FutureImpl
+    private MvccEntitySerializationStrategy serializationStrategy;
+
+
+    @Override
+    protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
+        return serializationStrategy;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
new file mode 100644
index 0000000..795ff9a
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
@@ -0,0 +1,108 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
+import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Classy class class.
+ */
+public abstract class MvccEntitySerializationStrategyV3Test extends MvccEntitySerializationStrategyImplTest{
+    @Inject
+    protected SerializationFig serializationFig;
+
+    @Inject
+    protected CassandraFig cassandraFig;
+
+    private int setMaxEntitySize;
+
+    @Before
+    public void setUp() {
+        setMaxEntitySize = serializationFig.getMaxEntitySize();
+    }
+
+
+    @After
+    public void tearDown() {
+        SetConfigTestBypass.setValueByPass(serializationFig, "getMaxEntitySize", setMaxEntitySize + "");
+    }
+
+    /**
+     * Tests an entity with more than  65535 bytes worth of data is successfully stored and retrieved
+     */
+    @Test
+    public void largeEntityWriteRead() throws ConnectionException {
+        final int setSize = 65535 * 2;
+
+
+        //this is the size it works out to be when serialized, we want to allow this size
+
+        SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", 65535 * 10 + "" );
+        final Entity entity = EntityHelper.generateEntity(setSize);
+
+        //now we have one massive, entity, save it and retrieve it.
+        CollectionScope context =
+                new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "parent" ), "tests" );
+
+
+        final Id id = entity.getId();
+        ValidationUtils.verifyIdentity(id);
+        final UUID version = UUIDGenerator.newTimeUUID();
+        final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
+
+        final MvccEntity mvccEntity = new MvccEntityImpl( id, version, status, entity );
+
+
+        getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
+
+        //now load it
+        final Iterator<MvccEntity> loaded =
+                getMvccEntitySerializationStrategy().loadDescendingHistory( context, id, version, 100 );
+
+
+        assertTrue( loaded.hasNext() );
+
+        final MvccEntity loadedEntity = loaded.next();
+
+        assertLargeEntity( mvccEntity, loadedEntity );
+
+
+        MvccEntity returned =
+                serializationStrategy.load( context, Collections.singleton(id), version ).getEntity( id );
+
+        assertLargeEntity( mvccEntity, returned );
+    }
+
+
+
+    protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+
+        org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
+
+        EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/FutureImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/FutureImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/FutureImpl.java
new file mode 100644
index 0000000..aac6c7f
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/FutureImpl.java
@@ -0,0 +1,19 @@
+package org.apache.usergrid.persistence.core.guice;
+
+import com.google.inject.BindingAnnotation;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Represents 2 versions of an impl.  Generally used for online migration.  This represents the version that is the
+ * current version of the implementation.  I.E the "new" version.
+ */
+@BindingAnnotation
+@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+public @interface FutureImpl {}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
index c5089dc..f407239 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
@@ -19,13 +19,19 @@
 package org.apache.usergrid.persistence.model.entity;
 
 
-import java.util.UUID;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
 
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.persistence.model.field.*;
 import org.apache.usergrid.persistence.model.field.value.EntityObject;
 
 import com.fasterxml.jackson.annotation.*;
 import com.google.common.base.Preconditions;
+import org.apache.usergrid.persistence.model.field.value.Location;
 
 
 
@@ -37,6 +43,8 @@ import com.google.common.base.Preconditions;
 @JsonTypeInfo( use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class" )
 public class Entity extends EntityObject {
 
+    private static MapToEntityConverter mapToEntityConverter = new MapToEntityConverter();
+
     /**
      * The id.  We should never serialize this
      */
@@ -70,6 +78,13 @@ public class Entity extends EntityObject {
         this(new SimpleId( type ));
     }
 
+    /**
+     * Generate an entity based on the map
+     * @param map
+     */
+    public Entity(EntityMap map){
+        mapToEntityConverter.fromMap(this,map,true);
+    }
 
     /**
      * Do not use!  This is only for serialization.
@@ -131,4 +146,5 @@ public class Entity extends EntityObject {
                 ", version=" + version +
                 '}';
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
new file mode 100644
index 0000000..eeac6bc
--- /dev/null
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
@@ -0,0 +1,24 @@
+package org.apache.usergrid.persistence.model.entity;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * Core persistence Entity Map structure to persist to
+ */
+public class EntityMap extends HashMap<String,Object> {
+    private static EntityToMapConverter entityToMapConverter = new EntityToMapConverter();
+
+    public EntityMap(){
+        super();
+    }
+
+    public EntityMap(EntityObject entityObject){
+        entityToMapConverter.toMap(entityObject, this);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
new file mode 100644
index 0000000..9b67b49
--- /dev/null
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.model.entity;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * abstract conversion to Map<String,Object> form EntityObject
+ */
+public class EntityToMapConverter{
+    public static ObjectMapper objectMapper = new ObjectMapper(  );
+    /**
+     * Convert Entity to Map, adding version_ug_field and a {name}_ug_analyzed field for each
+     * StringField.
+     */
+
+    public Map toMap(EntityObject entity) {
+        Map<String,Object> map = new HashMap<>();
+        return toMap(entity,map);
+    }
+    public Map toMap(EntityObject entity,Map<String,Object> entityMap) {
+
+        for (Object f : entity.getFields().toArray()) {
+            Field field = (Field) f;
+
+            if (f instanceof ListField || f instanceof ArrayField) {
+                List list = (List) field.getValue();
+                entityMap.put(field.getName(),
+                        new ArrayList( processCollectionForMap(list)));
+
+            } else if (f instanceof SetField) {
+                Set set = (Set) field.getValue();
+                entityMap.put(field.getName(),
+                        new ArrayList( processCollectionForMap(set)));
+
+            } else if (f instanceof EntityObjectField) {
+                EntityObject eo = (EntityObject) field.getValue();
+                entityMap.put( field.getName(), toMap(eo)); // recursion
+
+            } else if (f instanceof StringField) {
+                entityMap.put(field.getName(), ((String) field.getValue()));
+
+            } else if (f instanceof LocationField) {
+                LocationField locField = (LocationField) f;
+                Map<String, Object> locMap = new HashMap<String, Object>();
+
+                // field names lat and lon trigger ElasticSearch geo location
+                locMap.put("lat", locField.getValue().getLatitude());
+                locMap.put("lon", locField.getValue().getLongitude());
+                entityMap.put( field.getName(), field.getValue());
+
+            } else if (f instanceof ByteArrayField) {
+                ByteArrayField bf = ( ByteArrayField ) f;
+
+                byte[] serilizedObj =  bf.getValue();
+                Object o;
+                try {
+                    o = objectMapper.readValue( serilizedObj, bf.getClassinfo() );
+                }
+                catch ( IOException e ) {
+                    throw new RuntimeException( "Can't deserialize object ",e );
+                }
+                entityMap.put( bf.getName(), o );
+            }
+            else {
+                entityMap.put( field.getName(), field.getValue());
+            }
+        }
+
+        return entityMap;
+    }
+    private Collection processCollectionForMap(Collection c) {
+        if (c.isEmpty()) {
+            return c;
+        }
+        List processed = new ArrayList();
+        Object sample = c.iterator().next();
+
+        if (sample instanceof Entity) {
+            for (Object o : c.toArray()) {
+                Entity e = (Entity) o;
+                processed.add(toMap(e));
+            }
+
+        } else if (sample instanceof List) {
+            for (Object o : c.toArray()) {
+                List list = (List) o;
+                processed.add(processCollectionForMap(list)); // recursion;
+            }
+
+        } else if (sample instanceof Set) {
+            for (Object o : c.toArray()) {
+                Set set = (Set) o;
+                processed.add(processCollectionForMap(set)); // recursion;
+            }
+
+        } else {
+            for (Object o : c.toArray()) {
+                processed.add(o);
+            }
+        }
+        return processed;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
new file mode 100644
index 0000000..0b76812
--- /dev/null
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
@@ -0,0 +1,195 @@
+package org.apache.usergrid.persistence.model.entity;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.value.Location;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+public class MapToEntityConverter{
+    private ObjectMapper objectMapper = new ObjectMapper();
+
+    public  Entity fromMap( Map<String, Object> map,  boolean topLevel ) {
+        return fromMap( null, map, topLevel );
+    }
+
+    public Entity fromMap(
+            Entity entity, Map<String, Object> map,  boolean topLevel ) {
+
+        if ( entity == null ) {
+            entity = new Entity();
+        }
+
+        for ( String fieldName : map.keySet() ) {
+
+            Object value = map.get(fieldName);
+
+            if ( value instanceof String ) {
+                entity.setField( new StringField( fieldName, (String)value, topLevel ));
+
+            } else if ( value instanceof Boolean ) {
+                entity.setField( new BooleanField( fieldName, (Boolean)value, topLevel ));
+
+            } else if ( value instanceof Integer ) {
+                entity.setField( new IntegerField( fieldName, (Integer)value, topLevel ));
+
+            } else if ( value instanceof Double ) {
+                entity.setField( new DoubleField( fieldName, (Double)value, topLevel ));
+
+            } else if ( value instanceof Float ) {
+                entity.setField( new FloatField( fieldName, (Float)value, topLevel ));
+
+            } else if ( value instanceof Long ) {
+                entity.setField( new LongField( fieldName, (Long)value, topLevel ));
+
+            } else if ( value instanceof List) {
+                entity.setField( listToListField( fieldName, (List)value ));
+
+            } else if ( value instanceof UUID) {
+                entity.setField( new UUIDField( fieldName, (UUID)value, topLevel ));
+
+            } else if ( value instanceof Map ) {
+                processMapValue( value, fieldName, entity);
+
+            } else if ( value instanceof Enum ) {
+                entity.setField( new StringField( fieldName, value.toString(), topLevel ));
+
+            } else if ( value != null ) {
+                byte[] valueSerialized;
+                try {
+                    valueSerialized = objectMapper.writeValueAsBytes( value );
+                }
+                catch ( JsonProcessingException e ) {
+                    throw new RuntimeException( "Can't serialize object ",e );
+                }
+                catch ( IOException e ) {
+                    throw new RuntimeException( "Can't serialize object ",e );
+                }
+                ByteBuffer byteBuffer = ByteBuffer.wrap( valueSerialized );
+                ByteArrayField bf = new ByteArrayField( fieldName, byteBuffer.array(), value.getClass() );
+                entity.setField( bf );
+            }
+        }
+
+        return entity;
+    }
+
+    private  ListField listToListField( String fieldName, List list ) {
+
+        if (list.isEmpty()) {
+            return new ArrayField( fieldName );
+        }
+
+        Object sample = list.get(0);
+
+        if ( sample instanceof Map ) {
+            return new ArrayField( fieldName, processListForField( list ));
+
+        } else if ( sample instanceof List ) {
+            return new ArrayField<List>( fieldName, processListForField( list ));
+
+        } else if ( sample instanceof String ) {
+            return new ArrayField<String>( fieldName, (List<String>)list );
+
+        } else if ( sample instanceof Boolean ) {
+            return new ArrayField<Boolean>( fieldName, (List<Boolean>)list );
+
+        } else if ( sample instanceof Integer ) {
+            return new ArrayField<Integer>( fieldName, (List<Integer>)list );
+
+        } else if ( sample instanceof Double ) {
+            return new ArrayField<Double>( fieldName, (List<Double>)list );
+
+        } else if ( sample instanceof Long ) {
+            return new ArrayField<Long>( fieldName, (List<Long>)list );
+
+        } else {
+            throw new RuntimeException("Unknown type " + sample.getClass().getName());
+        }
+    }
+    private  List processListForField( List list ) {
+        if ( list.isEmpty() ) {
+            return list;
+        }
+        Object sample = list.get(0);
+
+        if ( sample instanceof Map ) {
+            List<Entity> newList = new ArrayList<Entity>();
+            for ( Map<String, Object> map : (List<Map<String, Object>>)list ) {
+                newList.add( fromMap( map, false ) );
+            }
+            return newList;
+
+        } else if ( sample instanceof List ) {
+            return processListForField( list ); // recursion
+
+        } else {
+            return list;
+        }
+    }
+
+
+    private  void processMapValue(
+            Object value, String fieldName, Entity entity) {
+
+        // is the map really a location element?
+        if ("location" .equals(fieldName.toString().toLowerCase()) ) {
+
+            // get the object to inspect
+            Map<String, Object> origMap = (Map<String, Object>) value;
+            Map<String, Object> m = new HashMap<String, Object>();
+
+            // Tests expect us to treat "Longitude" the same as "longitude"
+            for ( String key : origMap.keySet() ) {
+                m.put( key.toLowerCase(), origMap.get(key) );
+            }
+
+            // Expect at least two fields in a Location object
+            if (m.size() >= 2) {
+
+                Double lat = null;
+                Double lon = null;
+
+                // check the properties to make sure they are set and are doubles
+                if (m.get("latitude") != null && m.get("longitude") != null) {
+                    try {
+                        lat = Double.parseDouble(m.get("latitude").toString());
+                        lon = Double.parseDouble(m.get("longitude").toString());
+
+                    } catch (NumberFormatException ignored) {
+                        throw new IllegalArgumentException(
+                                "Latitude and longitude must be doubles (e.g. 32.1234).");
+                    }
+                } else if (m.get("lat") != null && m.get("lon") != null) {
+                    try {
+                        lat = Double.parseDouble(m.get("lat").toString());
+                        lon = Double.parseDouble(m.get("lon").toString());
+                    } catch (NumberFormatException ignored) {
+                        throw new IllegalArgumentException(""
+                                + "Latitude and longitude must be doubles (e.g. 32.1234).");
+                    }
+                } else {
+                    throw new IllegalArgumentException("Location properties require two fields - "
+                            + "latitude and longitude, or lat and lon");
+                }
+
+                if (lat != null && lon != null) {
+                    entity.setField( new LocationField(fieldName, new Location(lat, lon)));
+                } else {
+                    throw new IllegalArgumentException( "Unable to parse location field properties "
+                            + "- make sure they conform - lat and lon, and should be doubles.");
+                }
+            } else {
+                throw new IllegalArgumentException("Location properties requires two fields - "
+                        + "latitude and longitude, or lat and lon.");
+            }
+        } else {
+            // not a location element, process it as map
+            entity.setField(new EntityObjectField(fieldName,
+                    fromMap((Map<String, Object>) value, false))); // recursion
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
index b746dc4..6675d73 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
@@ -25,7 +25,7 @@ import java.util.List;
  * An object field that represents a list of objects. This can also be used to represent arrays
  * @param <T> Type of entity in list, must be primitive or Entity.
  */
-public class ListField<T> extends AbstractField<List<T>> {
+public abstract class ListField<T> extends AbstractField<List<T>> {
 
     /**
      * Constructor that initializes with an empty set for adding to later

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
index 6701a7c..869aeac 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
@@ -78,7 +78,7 @@ class EntityIndexMapUtils {
 
             } else if ( value instanceof Boolean ) {
                 entity.setField( new BooleanField( fieldName, (Boolean)value ));
-                        
+
             } else if ( value instanceof Integer ) {
                 entity.setField( new IntegerField( fieldName, (Integer)value ));
 
@@ -87,13 +87,13 @@ class EntityIndexMapUtils {
 
 		    } else if ( value instanceof Float ) {
                 entity.setField( new FloatField( fieldName, (Float)value ));
-				
+
             } else if ( value instanceof Long ) {
                 entity.setField( new LongField( fieldName, (Long)value ));
 
             } else if ( value instanceof List) {
-                entity.setField( listToListField( fieldName, (List)value ));  
-            
+                entity.setField( listToListField( fieldName, (List)value ));
+
             } else if ( value instanceof UUID) {
                 entity.setField( new UUIDField( fieldName, (UUID)value ));
 
@@ -113,7 +113,7 @@ class EntityIndexMapUtils {
 							lat = Double.parseDouble( m.get("latitude").toString() );
 							lon = Double.parseDouble( m.get("longitude").toString() );
 
-						} else if ( m.get("lat") != null && m.get("lon") != null ) { 
+						} else if ( m.get("lat") != null && m.get("lon") != null ) {
 							lat = Double.parseDouble( m.get("lat").toString() );
 							lon = Double.parseDouble( m.get("lon").toString() );
 						}
@@ -124,16 +124,16 @@ class EntityIndexMapUtils {
 					}
 				}
 
-				if ( field == null ) { 
+				if ( field == null ) {
 
 					// not a location element, process it as map
-					entity.setField( new EntityObjectField( fieldName, 
+					entity.setField( new EntityObjectField( fieldName,
 						fromMap( (Map<String, Object>)value ))); // recursion
 
 				} else {
 					entity.setField( field );
 				}
-	
+
 			} else if ( value instanceof Object) {
 
                 byte[] valueSerialized;
@@ -158,42 +158,42 @@ class EntityIndexMapUtils {
         return entity;
     }
 
-    
+
     private static ListField listToListField( String fieldName, List list ) {
 
         if (list.isEmpty()) {
-            return new ListField( fieldName );
+            return new ArrayField( fieldName );
         }
 
         Object sample = list.get(0);
 
         if ( sample instanceof Map ) {
-            return new ListField<Entity>( fieldName, processListForField( list ));
+            return new ArrayField( fieldName, processListForField( list ));
 
         } else if ( sample instanceof List ) {
-            return new ListField<List>( fieldName, processListForField( list ));
-            
+            return new ArrayField<List>( fieldName, processListForField( list ));
+
         } else if ( sample instanceof String ) {
-            return new ListField<String>( fieldName, (List<String>)list );
-                    
+            return new ArrayField<String>( fieldName, (List<String>)list );
+
         } else if ( sample instanceof Boolean ) {
-            return new ListField<Boolean>( fieldName, (List<Boolean>)list );
-                    
+            return new ArrayField<Boolean>( fieldName, (List<Boolean>)list );
+
         } else if ( sample instanceof Integer ) {
-            return new ListField<Integer>( fieldName, (List<Integer>)list );
+            return new ArrayField<Integer>( fieldName, (List<Integer>)list );
 
         } else if ( sample instanceof Double ) {
-            return new ListField<Double>( fieldName, (List<Double>)list );
+            return new ArrayField<Double>( fieldName, (List<Double>)list );
 
         } else if ( sample instanceof Long ) {
-            return new ListField<Long>( fieldName, (List<Long>)list );
+            return new ArrayField<Long>( fieldName, (List<Long>)list );
 
         } else {
             throw new RuntimeException("Unknown type " + sample.getClass().getName());
         }
     }
 
-    
+
     private static List processListForField( List list ) {
         if ( list.isEmpty() ) {
             return list;
@@ -209,10 +209,10 @@ class EntityIndexMapUtils {
 
         } else if ( sample instanceof List ) {
             return processListForField( list ); // recursion
-            
-        } else { 
+
+        } else {
             return list;
-        } 
+        }
     }
 
 
@@ -245,7 +245,7 @@ class EntityIndexMapUtils {
                 LocationField locField = (LocationField) f;
                 Map<String, Object> locMap = new HashMap<String, Object>();
 
-                // field names lat and lon trigger ElasticSearch geo location 
+                // field names lat and lon trigger ElasticSearch geo location
                 locMap.put("lat", locField.getValue().getLatitude());
                 locMap.put("lon", locField.getValue().getLongitude());
                 entityMap.put( field.getName(), locMap);
@@ -272,7 +272,7 @@ class EntityIndexMapUtils {
         return entityMap;
     }
 
-    
+
     private static Collection processCollectionForMap(Collection c) {
         if (c.isEmpty()) {
             return c;