You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/01/29 21:34:19 UTC
incubator-usergrid git commit: added new entity object for
serialization
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-365 198f4795a -> 481ea5e9b
added new entity object for serialization
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/481ea5e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/481ea5e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/481ea5e9
Branch: refs/heads/USERGRID-365
Commit: 481ea5e9bd26f082f634f04032f2836278e58684
Parents: 198f479
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Jan 29 11:46:01 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Jan 29 11:46:01 2015 -0700
----------------------------------------------------------------------
.../MvccEntitySerializationStrategyV3Impl.java | 212 +++++++++++++++++++
.../serialization/impl/SerializationModule.java | 3 +
...ccEntitySerializationStrategyV3ImplTest.java | 33 +++
.../MvccEntitySerializationStrategyV3Test.java | 108 ++++++++++
.../persistence/core/guice/FutureImpl.java | 19 ++
.../persistence/model/entity/Entity.java | 18 +-
.../persistence/model/entity/EntityMap.java | 24 +++
.../model/entity/EntityToMapConverter.java | 125 +++++++++++
.../model/entity/MapToEntityConverter.java | 195 +++++++++++++++++
.../persistence/model/field/ListField.java | 2 +-
.../index/impl/EntityIndexMapUtils.java | 52 ++---
11 files changed, 763 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
new file mode 100644
index 0000000..ce8dbef
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Impl.java
@@ -0,0 +1,212 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+import com.netflix.astyanax.serializers.UUIDSerializer;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
+import org.apache.usergrid.persistence.collection.exception.EntityTooLargeException;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.*;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.EntityMap;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.UUID;
+
+/**
+ * Classy class class.
+ */
+public class MvccEntitySerializationStrategyV3Impl extends MvccEntitySerializationStrategyImpl {
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ private static final CollectionScopedRowKeySerializer<Id> ROW_KEY_SER =
+ new CollectionScopedRowKeySerializer<>(ID_SER);
+
+
+ private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> CF_ENTITY_DATA =
+ new MultiTennantColumnFamily<>("Entity_Version_Data_V2", ROW_KEY_SER, UUIDSerializer.get());
+
+ private static final FieldBufferSerializer FIELD_BUFFER_SERIALIZER = FieldBufferSerializer.get();
+
+
+ private final EntitySerializer entitySerializer;
+
+
+ @Inject
+ public MvccEntitySerializationStrategyV3Impl(final Keyspace keyspace, final SerializationFig serializationFig, final CassandraFig cassandraFig) {
+ super(keyspace, serializationFig, cassandraFig);
+ entitySerializer = new EntitySerializer(serializationFig);
+ }
+
+
+ @Override
+ protected AbstractSerializer<MvccEntitySerializationStrategyImpl.EntityWrapper> getEntitySerializer() {
+ return entitySerializer;
+ }
+
+
+ @Override
+ protected MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> getColumnFamily() {
+ return CF_ENTITY_DATA;
+ }
+
+
+ /**
+ * We should only ever create this once, since this impl is a singleton
+ */
+ public final class EntitySerializer extends AbstractSerializer<MvccEntitySerializationStrategyImpl.EntityWrapper> {
+
+
+ private final SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ private final ObjectMapper MAPPER = new ObjectMapper(SMILE_FACTORY);
+
+
+ private SerializationFig serializationFig;
+
+
+ private byte STATE_COMPLETE = 0;
+ private byte STATE_DELETED = 1;
+ private byte STATE_PARTIAL = 2;
+
+ private byte VERSION = 1;
+
+
+ public EntitySerializer(final SerializationFig serializationFig) {
+ this.serializationFig = serializationFig;
+
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
+ // causes slowness
+ MAPPER.enableDefaultTypingAsProperty(ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class");
+ }
+
+
+ @Override
+ public ByteBuffer toByteBuffer(final MvccEntitySerializationStrategyImpl.EntityWrapper wrapper) {
+ if (wrapper == null) {
+ return null;
+ }
+
+ //we always have a max of 3 fields
+ FieldBufferBuilder builder = new FieldBufferBuilder(3);
+
+ builder.addByte(VERSION);
+
+ //mark this version as empty
+ if (!wrapper.entity.isPresent()) {
+ //we're empty
+ builder.addByte(STATE_DELETED);
+
+
+ return FIELD_BUFFER_SERIALIZER.toByteBuffer(builder.build());
+ }
+
+ //we have an entity
+
+ if (wrapper.status == MvccEntity.Status.COMPLETE) {
+ builder.addByte(STATE_COMPLETE);
+ } else {
+ builder.addByte(STATE_PARTIAL);
+ }
+
+
+ //Get Entity
+ final Entity entity = wrapper.entity.get();
+ //Convert to internal entity map
+ final EntityMap entityMap = new EntityMap(entity);
+ final byte[] entityBytes;
+ try {
+ entityBytes = MAPPER.writeValueAsBytes(entityMap);
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to serialize entity", e);
+ }
+
+
+ final int maxEntrySize = serializationFig.getMaxEntitySize();
+
+ if (entityBytes.length > maxEntrySize) {
+ throw new EntityTooLargeException(entity, maxEntrySize, entityBytes.length,
+ "Your entity cannot exceed " + maxEntrySize + " bytes. The entity you tried to save was "
+ + entityBytes.length + " bytes");
+ }
+
+ builder.addBytes(entityBytes);
+
+ return FIELD_BUFFER_SERIALIZER.toByteBuffer(builder.build());
+ }
+
+
+ @Override
+ public MvccEntitySerializationStrategyImpl.EntityWrapper fromByteBuffer(final ByteBuffer byteBuffer) {
+
+ /**
+ * We intentionally turn data corruption exceptions when we're unable to de-serialize
+ * the data in cassandra. If this occurs, we'll never be able to de-serialize it
+ * and it should be considered lost. This is an error that is occurring due to a bug
+ * in serializing the entity. This is a lazy recognition + repair signal for deployment with
+ * existing systems.
+ */
+
+ final FieldBuffer fieldBuffer;
+
+ try {
+ fieldBuffer = FIELD_BUFFER_SERIALIZER.fromByteBuffer(byteBuffer);
+ } catch (Exception e) {
+ throw new DataCorruptionException("Unable to de-serialze entity", e);
+ }
+
+ FieldBufferParser parser = new FieldBufferParser(fieldBuffer);
+
+
+ byte version = parser.readByte();
+
+ if (VERSION != version) {
+ throw new UnsupportedOperationException("A version of type " + version + " is unsupported");
+ }
+
+ byte state = parser.readByte();
+
+ // it's been deleted, remove it
+
+ if (STATE_DELETED == state) {
+ return new MvccEntitySerializationStrategyImpl.EntityWrapper(MvccEntity.Status.COMPLETE, Optional.<Entity>absent());
+ }
+
+ EntityMap storedEntity;
+
+ byte[] array = parser.readBytes();
+
+ try {
+ storedEntity = MAPPER.readValue(array, EntityMap.class);
+ } catch (Exception e) {
+ throw new DataCorruptionException("Unable to read entity data", e);
+ }
+
+ Entity entityObject = new Entity(storedEntity);
+
+ final Optional<Entity> entity = Optional.of(entityObject);
+
+ if (STATE_COMPLETE == state) {
+ return new MvccEntitySerializationStrategyImpl.EntityWrapper(MvccEntity.Status.COMPLETE, entity);
+ }
+
+ // it's partial by default
+ return new MvccEntitySerializationStrategyImpl.EntityWrapper(MvccEntity.Status.PARTIAL, entity);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index 33a092d..ac2b48c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -22,6 +22,7 @@ import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationSt
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.FutureImpl;
import org.apache.usergrid.persistence.core.guice.PreviousImpl;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
@@ -47,6 +48,8 @@ public class SerializationModule extends AbstractModule {
.to( MvccEntitySerializationStrategyV1Impl.class );
bind( MvccEntitySerializationStrategy.class ).annotatedWith( CurrentImpl.class )
.to( MvccEntitySerializationStrategyV2Impl.class );
+ bind( MvccEntitySerializationStrategy.class ).annotatedWith( FutureImpl.class )
+ .to(MvccEntitySerializationStrategyV3Impl.class);
bind( MvccEntitySerializationStrategy.class ).annotatedWith( ProxyImpl.class )
.to( MvccEntitySerializationStrategyProxyImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
new file mode 100644
index 0000000..4a3d761
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3ImplTest.java
@@ -0,0 +1,33 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
+import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.FutureImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.junit.runner.RunWith;
+
+/**
+ * Classy class class.
+ */
+
+
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class MvccEntitySerializationStrategyV3ImplTest extends MvccEntitySerializationStrategyV3Test {
+ @Inject
+ @FutureImpl
+ private MvccEntitySerializationStrategy serializationStrategy;
+
+
+ @Override
+ protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
+ return serializationStrategy;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
new file mode 100644
index 0000000..795ff9a
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV3Test.java
@@ -0,0 +1,108 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
+import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Classy class class.
+ */
+public abstract class MvccEntitySerializationStrategyV3Test extends MvccEntitySerializationStrategyImplTest{
+ @Inject
+ protected SerializationFig serializationFig;
+
+ @Inject
+ protected CassandraFig cassandraFig;
+
+ private int setMaxEntitySize;
+
+ @Before
+ public void setUp() {
+ setMaxEntitySize = serializationFig.getMaxEntitySize();
+ }
+
+
+ @After
+ public void tearDown() {
+ SetConfigTestBypass.setValueByPass(serializationFig, "getMaxEntitySize", setMaxEntitySize + "");
+ }
+
+ /**
+ * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
+ */
+ @Test
+ public void largeEntityWriteRead() throws ConnectionException {
+ final int setSize = 65535 * 2;
+
+
+ //this is the size it works out to be when serialized, we want to allow this size
+
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", 65535 * 10 + "" );
+ final Entity entity = EntityHelper.generateEntity(setSize);
+
+ //now we have one massive, entity, save it and retrieve it.
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "parent" ), "tests" );
+
+
+ final Id id = entity.getId();
+ ValidationUtils.verifyIdentity(id);
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
+
+ final MvccEntity mvccEntity = new MvccEntityImpl( id, version, status, entity );
+
+
+ getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
+
+ //now load it
+ final Iterator<MvccEntity> loaded =
+ getMvccEntitySerializationStrategy().loadDescendingHistory( context, id, version, 100 );
+
+
+ assertTrue( loaded.hasNext() );
+
+ final MvccEntity loadedEntity = loaded.next();
+
+ assertLargeEntity( mvccEntity, loadedEntity );
+
+
+ MvccEntity returned =
+ serializationStrategy.load( context, Collections.singleton(id), version ).getEntity( id );
+
+ assertLargeEntity( mvccEntity, returned );
+ }
+
+
+
+ protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+
+ org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
+
+ EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/FutureImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/FutureImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/FutureImpl.java
new file mode 100644
index 0000000..aac6c7f
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guice/FutureImpl.java
@@ -0,0 +1,19 @@
+package org.apache.usergrid.persistence.core.guice;
+
+import com.google.inject.BindingAnnotation;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+import static java.lang.annotation.ElementType.FIELD;
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.PARAMETER;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * Represents 2 versions of an impl. Generally used for online migration. This represents the version that is the
+ * current version of the implementation. I.E the "new" version.
+ */
+@BindingAnnotation
+@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME)
+public @interface FutureImpl {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
index c5089dc..f407239 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Entity.java
@@ -19,13 +19,19 @@
package org.apache.usergrid.persistence.model.entity;
-import java.util.UUID;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.persistence.model.field.*;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import com.fasterxml.jackson.annotation.*;
import com.google.common.base.Preconditions;
+import org.apache.usergrid.persistence.model.field.value.Location;
@@ -37,6 +43,8 @@ import com.google.common.base.Preconditions;
@JsonTypeInfo( use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class" )
public class Entity extends EntityObject {
+ private static MapToEntityConverter mapToEntityConverter = new MapToEntityConverter();
+
/**
* The id. We should never serialize this
*/
@@ -70,6 +78,13 @@ public class Entity extends EntityObject {
this(new SimpleId( type ));
}
+ /**
+ * Generate an entity based on the map
+ * @param map
+ */
+ public Entity(EntityMap map){
+ mapToEntityConverter.fromMap(this,map,true);
+ }
/**
* Do not use! This is only for serialization.
@@ -131,4 +146,5 @@ public class Entity extends EntityObject {
", version=" + version +
'}';
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
new file mode 100644
index 0000000..eeac6bc
--- /dev/null
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityMap.java
@@ -0,0 +1,24 @@
+package org.apache.usergrid.persistence.model.entity;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * Core persistence Entity Map structure to persist to
+ */
+public class EntityMap extends HashMap<String,Object> {
+ private static EntityToMapConverter entityToMapConverter = new EntityToMapConverter();
+
+ public EntityMap(){
+ super();
+ }
+
+ public EntityMap(EntityObject entityObject){
+ entityToMapConverter.toMap(entityObject, this);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
new file mode 100644
index 0000000..9b67b49
--- /dev/null
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/EntityToMapConverter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.model.entity;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * abstract conversion to Map<String,Object> form EntityObject
+ */
+public class EntityToMapConverter{
+ public static ObjectMapper objectMapper = new ObjectMapper( );
+ /**
+ * Convert Entity to Map, adding version_ug_field and a {name}_ug_analyzed field for each
+ * StringField.
+ */
+
+ public Map toMap(EntityObject entity) {
+ Map<String,Object> map = new HashMap<>();
+ return toMap(entity,map);
+ }
+ public Map toMap(EntityObject entity,Map<String,Object> entityMap) {
+
+ for (Object f : entity.getFields().toArray()) {
+ Field field = (Field) f;
+
+ if (f instanceof ListField || f instanceof ArrayField) {
+ List list = (List) field.getValue();
+ entityMap.put(field.getName(),
+ new ArrayList( processCollectionForMap(list)));
+
+ } else if (f instanceof SetField) {
+ Set set = (Set) field.getValue();
+ entityMap.put(field.getName(),
+ new ArrayList( processCollectionForMap(set)));
+
+ } else if (f instanceof EntityObjectField) {
+ EntityObject eo = (EntityObject) field.getValue();
+ entityMap.put( field.getName(), toMap(eo)); // recursion
+
+ } else if (f instanceof StringField) {
+ entityMap.put(field.getName(), ((String) field.getValue()));
+
+ } else if (f instanceof LocationField) {
+ LocationField locField = (LocationField) f;
+ Map<String, Object> locMap = new HashMap<String, Object>();
+
+ // field names lat and lon trigger ElasticSearch geo location
+ locMap.put("lat", locField.getValue().getLatitude());
+ locMap.put("lon", locField.getValue().getLongitude());
+ entityMap.put( field.getName(), field.getValue());
+
+ } else if (f instanceof ByteArrayField) {
+ ByteArrayField bf = ( ByteArrayField ) f;
+
+ byte[] serilizedObj = bf.getValue();
+ Object o;
+ try {
+ o = objectMapper.readValue( serilizedObj, bf.getClassinfo() );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Can't deserialize object ",e );
+ }
+ entityMap.put( bf.getName(), o );
+ }
+ else {
+ entityMap.put( field.getName(), field.getValue());
+ }
+ }
+
+ return entityMap;
+ }
+ private Collection processCollectionForMap(Collection c) {
+ if (c.isEmpty()) {
+ return c;
+ }
+ List processed = new ArrayList();
+ Object sample = c.iterator().next();
+
+ if (sample instanceof Entity) {
+ for (Object o : c.toArray()) {
+ Entity e = (Entity) o;
+ processed.add(toMap(e));
+ }
+
+ } else if (sample instanceof List) {
+ for (Object o : c.toArray()) {
+ List list = (List) o;
+ processed.add(processCollectionForMap(list)); // recursion;
+ }
+
+ } else if (sample instanceof Set) {
+ for (Object o : c.toArray()) {
+ Set set = (Set) o;
+ processed.add(processCollectionForMap(set)); // recursion;
+ }
+
+ } else {
+ for (Object o : c.toArray()) {
+ processed.add(o);
+ }
+ }
+ return processed;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
new file mode 100644
index 0000000..0b76812
--- /dev/null
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
@@ -0,0 +1,195 @@
+package org.apache.usergrid.persistence.model.entity;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.usergrid.persistence.model.field.*;
+import org.apache.usergrid.persistence.model.field.value.Location;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.*;
+
+public class MapToEntityConverter{
+ private ObjectMapper objectMapper = new ObjectMapper();
+
+ public Entity fromMap( Map<String, Object> map, boolean topLevel ) {
+ return fromMap( null, map, topLevel );
+ }
+
+ public Entity fromMap(
+ Entity entity, Map<String, Object> map, boolean topLevel ) {
+
+ if ( entity == null ) {
+ entity = new Entity();
+ }
+
+ for ( String fieldName : map.keySet() ) {
+
+ Object value = map.get(fieldName);
+
+ if ( value instanceof String ) {
+ entity.setField( new StringField( fieldName, (String)value, topLevel ));
+
+ } else if ( value instanceof Boolean ) {
+ entity.setField( new BooleanField( fieldName, (Boolean)value, topLevel ));
+
+ } else if ( value instanceof Integer ) {
+ entity.setField( new IntegerField( fieldName, (Integer)value, topLevel ));
+
+ } else if ( value instanceof Double ) {
+ entity.setField( new DoubleField( fieldName, (Double)value, topLevel ));
+
+ } else if ( value instanceof Float ) {
+ entity.setField( new FloatField( fieldName, (Float)value, topLevel ));
+
+ } else if ( value instanceof Long ) {
+ entity.setField( new LongField( fieldName, (Long)value, topLevel ));
+
+ } else if ( value instanceof List) {
+ entity.setField( listToListField( fieldName, (List)value ));
+
+ } else if ( value instanceof UUID) {
+ entity.setField( new UUIDField( fieldName, (UUID)value, topLevel ));
+
+ } else if ( value instanceof Map ) {
+ processMapValue( value, fieldName, entity);
+
+ } else if ( value instanceof Enum ) {
+ entity.setField( new StringField( fieldName, value.toString(), topLevel ));
+
+ } else if ( value != null ) {
+ byte[] valueSerialized;
+ try {
+ valueSerialized = objectMapper.writeValueAsBytes( value );
+ }
+ catch ( JsonProcessingException e ) {
+ throw new RuntimeException( "Can't serialize object ",e );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Can't serialize object ",e );
+ }
+ ByteBuffer byteBuffer = ByteBuffer.wrap( valueSerialized );
+ ByteArrayField bf = new ByteArrayField( fieldName, byteBuffer.array(), value.getClass() );
+ entity.setField( bf );
+ }
+ }
+
+ return entity;
+ }
+
+ private ListField listToListField( String fieldName, List list ) {
+
+ if (list.isEmpty()) {
+ return new ArrayField( fieldName );
+ }
+
+ Object sample = list.get(0);
+
+ if ( sample instanceof Map ) {
+ return new ArrayField( fieldName, processListForField( list ));
+
+ } else if ( sample instanceof List ) {
+ return new ArrayField<List>( fieldName, processListForField( list ));
+
+ } else if ( sample instanceof String ) {
+ return new ArrayField<String>( fieldName, (List<String>)list );
+
+ } else if ( sample instanceof Boolean ) {
+ return new ArrayField<Boolean>( fieldName, (List<Boolean>)list );
+
+ } else if ( sample instanceof Integer ) {
+ return new ArrayField<Integer>( fieldName, (List<Integer>)list );
+
+ } else if ( sample instanceof Double ) {
+ return new ArrayField<Double>( fieldName, (List<Double>)list );
+
+ } else if ( sample instanceof Long ) {
+ return new ArrayField<Long>( fieldName, (List<Long>)list );
+
+ } else {
+ throw new RuntimeException("Unknown type " + sample.getClass().getName());
+ }
+ }
+ private List processListForField( List list ) {
+ if ( list.isEmpty() ) {
+ return list;
+ }
+ Object sample = list.get(0);
+
+ if ( sample instanceof Map ) {
+ List<Entity> newList = new ArrayList<Entity>();
+ for ( Map<String, Object> map : (List<Map<String, Object>>)list ) {
+ newList.add( fromMap( map, false ) );
+ }
+ return newList;
+
+ } else if ( sample instanceof List ) {
+ return processListForField( list ); // recursion
+
+ } else {
+ return list;
+ }
+ }
+
+
+ private void processMapValue(
+ Object value, String fieldName, Entity entity) {
+
+ // is the map really a location element?
+ if ("location" .equals(fieldName.toString().toLowerCase()) ) {
+
+ // get the object to inspect
+ Map<String, Object> origMap = (Map<String, Object>) value;
+ Map<String, Object> m = new HashMap<String, Object>();
+
+ // Tests expect us to treat "Longitude" the same as "longitude"
+ for ( String key : origMap.keySet() ) {
+ m.put( key.toLowerCase(), origMap.get(key) );
+ }
+
+ // Expect at least two fields in a Location object
+ if (m.size() >= 2) {
+
+ Double lat = null;
+ Double lon = null;
+
+ // check the properties to make sure they are set and are doubles
+ if (m.get("latitude") != null && m.get("longitude") != null) {
+ try {
+ lat = Double.parseDouble(m.get("latitude").toString());
+ lon = Double.parseDouble(m.get("longitude").toString());
+
+ } catch (NumberFormatException ignored) {
+ throw new IllegalArgumentException(
+ "Latitude and longitude must be doubles (e.g. 32.1234).");
+ }
+ } else if (m.get("lat") != null && m.get("lon") != null) {
+ try {
+ lat = Double.parseDouble(m.get("lat").toString());
+ lon = Double.parseDouble(m.get("lon").toString());
+ } catch (NumberFormatException ignored) {
+ throw new IllegalArgumentException(""
+ + "Latitude and longitude must be doubles (e.g. 32.1234).");
+ }
+ } else {
+ throw new IllegalArgumentException("Location properties require two fields - "
+ + "latitude and longitude, or lat and lon");
+ }
+
+ if (lat != null && lon != null) {
+ entity.setField( new LocationField(fieldName, new Location(lat, lon)));
+ } else {
+ throw new IllegalArgumentException( "Unable to parse location field properties "
+ + "- make sure they conform - lat and lon, and should be doubles.");
+ }
+ } else {
+ throw new IllegalArgumentException("Location properties requires two fields - "
+ + "latitude and longitude, or lat and lon.");
+ }
+ } else {
+ // not a location element, process it as map
+ entity.setField(new EntityObjectField(fieldName,
+ fromMap((Map<String, Object>) value, false))); // recursion
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
index b746dc4..6675d73 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/ListField.java
@@ -25,7 +25,7 @@ import java.util.List;
* An object field that represents a list of objects. This can also be used to represent arrays
* @param <T> Type of entity in list, must be primitive or Entity.
*/
-public class ListField<T> extends AbstractField<List<T>> {
+public abstract class ListField<T> extends AbstractField<List<T>> {
/**
* Constructor that initializes with an empty set for adding to later
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/481ea5e9/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
index 6701a7c..869aeac 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexMapUtils.java
@@ -78,7 +78,7 @@ class EntityIndexMapUtils {
} else if ( value instanceof Boolean ) {
entity.setField( new BooleanField( fieldName, (Boolean)value ));
-
+
} else if ( value instanceof Integer ) {
entity.setField( new IntegerField( fieldName, (Integer)value ));
@@ -87,13 +87,13 @@ class EntityIndexMapUtils {
} else if ( value instanceof Float ) {
entity.setField( new FloatField( fieldName, (Float)value ));
-
+
} else if ( value instanceof Long ) {
entity.setField( new LongField( fieldName, (Long)value ));
} else if ( value instanceof List) {
- entity.setField( listToListField( fieldName, (List)value ));
-
+ entity.setField( listToListField( fieldName, (List)value ));
+
} else if ( value instanceof UUID) {
entity.setField( new UUIDField( fieldName, (UUID)value ));
@@ -113,7 +113,7 @@ class EntityIndexMapUtils {
lat = Double.parseDouble( m.get("latitude").toString() );
lon = Double.parseDouble( m.get("longitude").toString() );
- } else if ( m.get("lat") != null && m.get("lon") != null ) {
+ } else if ( m.get("lat") != null && m.get("lon") != null ) {
lat = Double.parseDouble( m.get("lat").toString() );
lon = Double.parseDouble( m.get("lon").toString() );
}
@@ -124,16 +124,16 @@ class EntityIndexMapUtils {
}
}
- if ( field == null ) {
+ if ( field == null ) {
// not a location element, process it as map
- entity.setField( new EntityObjectField( fieldName,
+ entity.setField( new EntityObjectField( fieldName,
fromMap( (Map<String, Object>)value ))); // recursion
} else {
entity.setField( field );
}
-
+
} else if ( value instanceof Object) {
byte[] valueSerialized;
@@ -158,42 +158,42 @@ class EntityIndexMapUtils {
return entity;
}
-
+
private static ListField listToListField( String fieldName, List list ) {
if (list.isEmpty()) {
- return new ListField( fieldName );
+ return new ArrayField( fieldName );
}
Object sample = list.get(0);
if ( sample instanceof Map ) {
- return new ListField<Entity>( fieldName, processListForField( list ));
+ return new ArrayField( fieldName, processListForField( list ));
} else if ( sample instanceof List ) {
- return new ListField<List>( fieldName, processListForField( list ));
-
+ return new ArrayField<List>( fieldName, processListForField( list ));
+
} else if ( sample instanceof String ) {
- return new ListField<String>( fieldName, (List<String>)list );
-
+ return new ArrayField<String>( fieldName, (List<String>)list );
+
} else if ( sample instanceof Boolean ) {
- return new ListField<Boolean>( fieldName, (List<Boolean>)list );
-
+ return new ArrayField<Boolean>( fieldName, (List<Boolean>)list );
+
} else if ( sample instanceof Integer ) {
- return new ListField<Integer>( fieldName, (List<Integer>)list );
+ return new ArrayField<Integer>( fieldName, (List<Integer>)list );
} else if ( sample instanceof Double ) {
- return new ListField<Double>( fieldName, (List<Double>)list );
+ return new ArrayField<Double>( fieldName, (List<Double>)list );
} else if ( sample instanceof Long ) {
- return new ListField<Long>( fieldName, (List<Long>)list );
+ return new ArrayField<Long>( fieldName, (List<Long>)list );
} else {
throw new RuntimeException("Unknown type " + sample.getClass().getName());
}
}
-
+
private static List processListForField( List list ) {
if ( list.isEmpty() ) {
return list;
@@ -209,10 +209,10 @@ class EntityIndexMapUtils {
} else if ( sample instanceof List ) {
return processListForField( list ); // recursion
-
- } else {
+
+ } else {
return list;
- }
+ }
}
@@ -245,7 +245,7 @@ class EntityIndexMapUtils {
LocationField locField = (LocationField) f;
Map<String, Object> locMap = new HashMap<String, Object>();
- // field names lat and lon trigger ElasticSearch geo location
+ // field names lat and lon trigger ElasticSearch geo location
locMap.put("lat", locField.getValue().getLatitude());
locMap.put("lon", locField.getValue().getLongitude());
entityMap.put( field.getName(), locMap);
@@ -272,7 +272,7 @@ class EntityIndexMapUtils {
return entityMap;
}
-
+
private static Collection processCollectionForMap(Collection c) {
if (c.isEmpty()) {
return c;