You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2014/11/21 22:35:39 UTC
[01/19] incubator-usergrid git commit: Added test to prove bug with
large entities
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o 74866a566 -> 0a92bee79
Added test to prove bug with large entities
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b6a83a4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b6a83a4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b6a83a4e
Branch: refs/heads/two-dot-o
Commit: b6a83a4ec3ea65e72093b4b97ba1e70dd2eb757b
Parents: 00eb339
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 18 18:06:09 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 18 18:06:09 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/LargeEntityIT.java | 568 +++++++++++++++++++
1 file changed, 568 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6a83a4e/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
new file mode 100644
index 0000000..4ce0b27
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
@@ -0,0 +1,568 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence;
+
+
+import java.util.Map;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.Application;
+import org.apache.usergrid.CoreApplication;
+import org.apache.usergrid.cassandra.Concurrent;
+import org.apache.usergrid.utils.JsonUtils;
+
+import static org.junit.Assert.assertEquals;
+
+
+@Concurrent()
+public class LargeEntityIT extends AbstractCoreIT {
+ private static final Logger LOG = LoggerFactory.getLogger( LargeEntityIT.class );
+
+ @Rule
+ public Application app = new CoreApplication( setup );
+
+
+ /**
+ * Tests creating a large entity, then loading it, modifying it, saving it, then loading it again
+ */
+ @Test
+ public void testLargeEntityCrud() throws Exception {
+
+ LOG.debug( "testLargeEntityCrud" );
+
+ final Map<String, Object> json = ( Map<String, Object> ) JsonUtils.parse( TEST_DATA );
+
+
+ final EntityManager em = app.getEntityManager();
+
+ final Entity createReturned = em.create( "test", json );
+
+ final Entity loadReturnedRef = em.get( createReturned );
+
+ final Entity loadReturnedId = em.get( createReturned.getUuid() );
+ assertEquals( createReturned, loadReturnedRef );
+ }
+
+
+ private static final String TEST_DATA =
+ "{\n" + " \"name\": \"p333812236\",\n" + " \"catentryid\": 7176,\n" + " \"productid\": 33381,\n"
+ + " \"services\": [\n" + " {\n" + " \"id\": 575,\n" + " \"name\": \"Monogram\",\n"
+ + " \"required\": 0,\n" + " \"attributes\": [\n" + " {\n"
+ + " \"id\": 80224,\n" + " \"name\": \"Style/Case\",\n"
+ + " \"value\": \"Block Uppercase\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 3,\n" + " \"label\": \"\",\n" + " \"maxlength\": 10,\n"
+ + " \"additionalvalidchars\": \"33,34,35,38,39,40,41,43,44,45,46,47,63,64\",\n"
+ + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 80225,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Block Upper- and Lowercase\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"label\": \"\",\n"
+ + " \"maxlength\": 10,\n"
+ + " \"additionalvalidchars\": \"33,34,35,38,39,40,41,43,44,45,46,47,63,64\",\n"
+ + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 80234,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Times Bold Uppercase\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"label\": \"\",\n"
+ + " \"maxlength\": 10,\n"
+ + " \"additionalvalidchars\": \"33,36,37,38,39,40,41,44,45,46,47,58,59,63\",\n"
+ + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 80235,\n"
+ + " \"name\": \"Style/Case\",\n"
+ + " \"value\": \"Times Bold Upper- and Lowercase\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 3,\n" + " \"label\": \"\",\n" + " \"maxlength\": 10,\n"
+ + " \"additionalvalidchars\": \"33,36,37,38,39,40,41,44,45,46,47,58,59,63\",\n"
+ + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 80237,\n"
+ + " \"name\": \"Monogram\",\n" + " \"value\": \"text\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 1,\n" + " \"label\": \"\",\n"
+ + " \"maxlength\": 10,\n" + " \"notes\": \"\"\n" + " },\n" + " {\n"
+ + " \"id\": 137788,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Black\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137789,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Navy\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137790,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Royal Blue\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137791,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Brown\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137792,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Purple\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137793,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Red\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137794,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Sea Green\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137795,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Hunter Green\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137796,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Dark Green\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137797,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Gold\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137798,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"White\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137799,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Platinum\",\n" + " \"isdefault\": 1,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137800,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Natural\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137801,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Taupe\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137802,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Light Purple\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137803,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Burgundy\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137804,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Teal\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137805,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Mimosa Yellow\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137806,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Turkey Red\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137807,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Lucerne Blue\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137808,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Nickel\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137809,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Orange\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137810,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Standard Blue\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137811,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Cobalt\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137812,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Dark Pink\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137813,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Sage\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137814,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Salmon\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137815,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Lime\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137816,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Rancho Rose\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137817,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Olive\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137818,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Rose\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137819,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Nile Green\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137820,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Dark Khaki\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137821,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Eggplant\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137822,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Cypress\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137823,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Terra Cotta\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137824,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Walnut\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137825,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Burlap\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 137826,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Loden\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 210983,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Copper\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 210984,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Dark Teal\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 216129,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Pink\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 227825,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Dark Brown\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 227826,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Emerald\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 227827,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Mid Blue\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 291205,\n" + " \"name\": \"Style/Case\",\n"
+ + " \"value\": \"Script Upper/Lower\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 3,\n" + " \"label\": \"\",\n" + " \"maxlength\": 10,\n"
+ + " \"additionalvalidchars\": \"33,34,35,39,43,44,46,47,63\",\n"
+ + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 291206,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Thin Script Upper/Lower\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"label\": \"\",\n"
+ + " \"maxlength\": 10,\n"
+ + " \"additionalvalidchars\": \"33,34,35,38,39,43,44,45,46,47,63\",\n"
+ + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 291207,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Athletic Upper\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"label\": \"\",\n"
+ + " \"maxlength\": 10,\n" + " \"notes\": \"\"\n" + " },\n" + " {\n"
+ + " \"id\": 353932,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Isle Green\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
+ + " \"id\": 368998,\n" + " \"name\": \"Thread Color\",\n"
+ + " \"value\": \"Plum\",\n" + " \"isdefault\": 0,\n"
+ + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " }\n" + " ]\n"
+ + " },\n" + " {\n" + " \"id\": 48071,\n"
+ + " \"name\": \"Monogram Single Large Letter\",\n" + " \"required\": 0,\n"
+ + " \"attributes\": [\n" + " {\n" + " \"id\": 291208,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Times Single Upper\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 1\n"
+ + " },\n" + " {\n" + " \"id\": 291209,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Heirloom Single Upper\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 1\n"
+ + " },\n" + " {\n" + " \"id\": 291210,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Athletic Single Upper\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 1\n"
+ + " },\n" + " {\n" + " \"id\": 291211,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Black\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291212,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Brown\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291213,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Burgundy\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291214,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Burlap\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291215,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Cobalt\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291216,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Copper\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291217,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Cypress\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291218,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Brown\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291219,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291220,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Khaki\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291221,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Pink\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291222,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Teal\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291223,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Eggplant\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291224,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Emerald\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291225,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Gold\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291226,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Hunter Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291227,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Light Purple\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291228,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Lime\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291229,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Loden\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291230,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Lucerne Blue\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291231,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Mid Blue\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291232,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Mimosa Yellow\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291233,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Natural\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291234,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Navy\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291235,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Nickel\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291236,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Nile Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291237,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Olive\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291238,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Orange\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291239,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Pink\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291240,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Platinum\",\n"
+ + " \"isdefault\": 1,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291241,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Purple\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291242,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Rancho Rose\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291243,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Red\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291244,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Rose\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291245,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Royal Blue\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291246,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Sage\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291247,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Salmon\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291248,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Sea Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291249,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Standard Blue\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291250,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Taupe\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291251,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Teal\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291252,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Terra Cotta\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291253,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Turkey Red\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291254,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Walnut\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291255,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"White\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291256,\n"
+ + " \"name\": \"Monogram Single Line 1\",\n" + " \"value\": \"text\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 1,\n" + " \"maxlength\": 1\n"
+ + " },\n" + " {\n" + " \"id\": 353933,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Isle Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 368999,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Plum\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " }\n" + " ]\n" + " },\n" + " {\n" + " \"id\": 48072,\n"
+ + " \"name\": \"Monogram 3-Letter\",\n" + " \"required\": 0,\n"
+ + " \"attributes\": [\n" + " {\n" + " \"id\": 291257,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Block Stack\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 1\n"
+ + " },\n" + " {\n" + " \"id\": 291258,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Flared\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 3\n"
+ + " },\n" + " {\n" + " \"id\": 291259,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Classic\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 3\n"
+ + " },\n" + " {\n" + " \"id\": 291260,\n"
+ + " \"name\": \"Style/Case\",\n" + " \"value\": \"Insignia\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 3\n"
+ + " },\n" + " {\n" + " \"id\": 291261,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Black\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291262,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Brown\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291263,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Burgundy\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291264,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Burlap\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291265,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Cobalt\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291266,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Copper\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291267,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Cypress\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291268,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Brown\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291269,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291270,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Khaki\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291271,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Pink\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291272,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Teal\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291273,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Eggplant\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291274,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Emerald\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291275,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Gold\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291276,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Hunter Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291277,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Light Purple\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291278,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Lime\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291279,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Loden\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291280,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Lucerne Blue\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291281,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Mid Blue\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291282,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Mimosa Yellow\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291283,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Natural\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291284,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Navy\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291285,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Nickel\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291286,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Nile Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291287,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Olive\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291288,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Orange\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291289,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Pink\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291290,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Platinum\",\n"
+ + " \"isdefault\": 1,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291291,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Purple\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291292,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Rancho Rose\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291293,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Red\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291294,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Rose\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291295,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Royal Blue\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291296,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Sage\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291297,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Salmon\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291298,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Sea Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291299,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Standard Blue\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291300,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Taupe\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291301,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Teal\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291302,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Terra Cotta\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291303,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Turkey Red\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291304,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Walnut\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291305,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"White\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 291306,\n"
+ + " \"name\": \"Monogram 3-Letter Line 1\",\n" + " \"value\": \"text\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 1,\n" + " \"maxlength\": 3\n"
+ + " },\n" + " {\n" + " \"id\": 353934,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Isle Green\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " },\n" + " {\n" + " \"id\": 369000,\n"
+ + " \"name\": \"Thread Color\",\n" + " \"value\": \"Plum\",\n"
+ + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
+ + " }\n" + " ]\n" + " },\n" + " {\n" + " \"id\": 3916,\n"
+ + " \"name\": \"Gift Boxing\",\n" + " \"required\": 0,\n" + " \"attributes\": [\n"
+ + " {\n" + " \"id\": 317426,\n" + " \"name\": \"Box Style\",\n"
+ + " \"value\": \"Tan/Green Ribbon\",\n" + " \"isdefault\": 1,\n"
+ + " \"sequence\": 1,\n" + " \"maxlength\": 0\n" + " }\n" + " ]\n"
+ + " }\n" + " ]\n" + "}";
+}
[02/19] incubator-usergrid git commit: Removed duplicate write call
in entity persist
Posted by sf...@apache.org.
Removed duplicate write call in entity persist
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3073fe81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3073fe81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3073fe81
Branch: refs/heads/two-dot-o
Commit: 3073fe811f97d95a311ebcd9a13eceb7b0a683e7
Parents: b6a83a4
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 19 14:11:07 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 19 14:11:07 2014 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/LargeEntityIT.java | 5 ++++-
.../collection/impl/EntityCollectionManagerImpl.java | 10 ----------
.../impl/MvccEntitySerializationStrategyImpl.java | 3 ++-
3 files changed, 6 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3073fe81/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
index 4ce0b27..0954ded 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
@@ -60,8 +60,11 @@ public class LargeEntityIT extends AbstractCoreIT {
final Entity loadReturnedRef = em.get( createReturned );
+ assertEquals( "Entities should be equal", createReturned, loadReturnedRef );
+
final Entity loadReturnedId = em.get( createReturned.getUuid() );
- assertEquals( createReturned, loadReturnedRef );
+
+ assertEquals( "Entities should be equal", createReturned, loadReturnedId );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3073fe81/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index bb75cc9..919e83b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -165,16 +165,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
// observable = Concurrent.concurrent( observable, Schedulers.io(), new WaitZip(),
// writeVerifyUnique, writeOptimisticVerify );
- observable.map(writeCommit).doOnNext(new Action1<Entity>() {
- @Override
- public void call(final Entity entity) {
- //TODO fire a task here
-
- //post-processing to come later. leave it empty for now.
- }
- }).doOnError(rollback);
-
-
// return the commit result.
return observable.map(writeCommit).doOnError(rollback);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3073fe81/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index b97429b..bbaeb4a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -478,7 +478,8 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
}
try {
- builder.addBytes( mapper.writeValueAsBytes( wrapper.entity.get() ) );
+ final byte[] entityBytes = mapper.writeValueAsBytes( wrapper.entity.get() ) ;
+ builder.addBytes( entityBytes );
}
catch ( Exception e ) {
throw new RuntimeException( "Unable to serialize entity", e );
[07/19] incubator-usergrid git commit: Fixed wiring problem with new
proxy impl
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bc0a9005/stack/core/src/test/resources/largeentity.json
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/largeentity.json b/stack/core/src/test/resources/largeentity.json
new file mode 100644
index 0000000..cb12d19
--- /dev/null
+++ b/stack/core/src/test/resources/largeentity.json
@@ -0,0 +1,1329 @@
+{
+ "name": "p333812236",
+ "catentryid": 7176,
+ "productid": 33381,
+ "services": [
+ {
+ "id": 575,
+ "name": "Monogram",
+ "required": 0,
+ "attributes": [
+ {
+ "id": 80224,
+ "name": "Style/Case",
+ "value": "Block Uppercase",
+ "isdefault": 0,
+ "sequence": 3,
+ "label": "",
+ "maxlength": 10,
+ "additionalvalidchars": "33,34,35,38,39,40,41,43,44,45,46,47,63,64",
+ "notes": ""
+ },
+ {
+ "id": 80225,
+ "name": "Style/Case",
+ "value": "Block Upper- and Lowercase",
+ "isdefault": 0,
+ "sequence": 3,
+ "label": "",
+ "maxlength": 10,
+ "additionalvalidchars": "33,34,35,38,39,40,41,43,44,45,46,47,63,64",
+ "notes": ""
+ },
+ {
+ "id": 80234,
+ "name": "Style/Case",
+ "value": "Times Bold Uppercase",
+ "isdefault": 0,
+ "sequence": 3,
+ "label": "",
+ "maxlength": 10,
+ "additionalvalidchars": "33,36,37,38,39,40,41,44,45,46,47,58,59,63",
+ "notes": ""
+ },
+ {
+ "id": 80235,
+ "name": "Style/Case",
+ "value": "Times Bold Upper- and Lowercase",
+ "isdefault": 0,
+ "sequence": 3,
+ "label": "",
+ "maxlength": 10,
+ "additionalvalidchars": "33,36,37,38,39,40,41,44,45,46,47,58,59,63",
+ "notes": ""
+ },
+ {
+ "id": 80237,
+ "name": "Monogram",
+ "value": "text",
+ "isdefault": 0,
+ "sequence": 1,
+ "label": "",
+ "maxlength": 10,
+ "notes": ""
+ },
+ {
+ "id": 137788,
+ "name": "Thread Color",
+ "value": "Black",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137789,
+ "name": "Thread Color",
+ "value": "Navy",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137790,
+ "name": "Thread Color",
+ "value": "Royal Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137791,
+ "name": "Thread Color",
+ "value": "Brown",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137792,
+ "name": "Thread Color",
+ "value": "Purple",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137793,
+ "name": "Thread Color",
+ "value": "Red",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137794,
+ "name": "Thread Color",
+ "value": "Sea Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137795,
+ "name": "Thread Color",
+ "value": "Hunter Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137796,
+ "name": "Thread Color",
+ "value": "Dark Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137797,
+ "name": "Thread Color",
+ "value": "Gold",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137798,
+ "name": "Thread Color",
+ "value": "White",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137799,
+ "name": "Thread Color",
+ "value": "Platinum",
+ "isdefault": 1,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137800,
+ "name": "Thread Color",
+ "value": "Natural",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137801,
+ "name": "Thread Color",
+ "value": "Taupe",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137802,
+ "name": "Thread Color",
+ "value": "Light Purple",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137803,
+ "name": "Thread Color",
+ "value": "Burgundy",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137804,
+ "name": "Thread Color",
+ "value": "Teal",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137805,
+ "name": "Thread Color",
+ "value": "Mimosa Yellow",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137806,
+ "name": "Thread Color",
+ "value": "Turkey Red",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137807,
+ "name": "Thread Color",
+ "value": "Lucerne Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137808,
+ "name": "Thread Color",
+ "value": "Nickel",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137809,
+ "name": "Thread Color",
+ "value": "Orange",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137810,
+ "name": "Thread Color",
+ "value": "Standard Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137811,
+ "name": "Thread Color",
+ "value": "Cobalt",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137812,
+ "name": "Thread Color",
+ "value": "Dark Pink",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137813,
+ "name": "Thread Color",
+ "value": "Sage",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137814,
+ "name": "Thread Color",
+ "value": "Salmon",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137815,
+ "name": "Thread Color",
+ "value": "Lime",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137816,
+ "name": "Thread Color",
+ "value": "Rancho Rose",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137817,
+ "name": "Thread Color",
+ "value": "Olive",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137818,
+ "name": "Thread Color",
+ "value": "Rose",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137819,
+ "name": "Thread Color",
+ "value": "Nile Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137820,
+ "name": "Thread Color",
+ "value": "Dark Khaki",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137821,
+ "name": "Thread Color",
+ "value": "Eggplant",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137822,
+ "name": "Thread Color",
+ "value": "Cypress",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137823,
+ "name": "Thread Color",
+ "value": "Terra Cotta",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137824,
+ "name": "Thread Color",
+ "value": "Walnut",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137825,
+ "name": "Thread Color",
+ "value": "Burlap",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 137826,
+ "name": "Thread Color",
+ "value": "Loden",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 210983,
+ "name": "Thread Color",
+ "value": "Copper",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 210984,
+ "name": "Thread Color",
+ "value": "Dark Teal",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 216129,
+ "name": "Thread Color",
+ "value": "Pink",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 227825,
+ "name": "Thread Color",
+ "value": "Dark Brown",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 227826,
+ "name": "Thread Color",
+ "value": "Emerald",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 227827,
+ "name": "Thread Color",
+ "value": "Mid Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291205,
+ "name": "Style/Case",
+ "value": "Script Upper/Lower",
+ "isdefault": 0,
+ "sequence": 3,
+ "label": "",
+ "maxlength": 10,
+ "additionalvalidchars": "33,34,35,39,43,44,46,47,63",
+ "notes": ""
+ },
+ {
+ "id": 291206,
+ "name": "Style/Case",
+ "value": "Thin Script Upper/Lower",
+ "isdefault": 0,
+ "sequence": 3,
+ "label": "",
+ "maxlength": 10,
+ "additionalvalidchars": "33,34,35,38,39,43,44,45,46,47,63",
+ "notes": ""
+ },
+ {
+ "id": 291207,
+ "name": "Style/Case",
+ "value": "Athletic Upper",
+ "isdefault": 0,
+ "sequence": 3,
+ "label": "",
+ "maxlength": 10,
+ "notes": ""
+ },
+ {
+ "id": 353932,
+ "name": "Thread Color",
+ "value": "Isle Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 368998,
+ "name": "Thread Color",
+ "value": "Plum",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ }
+ ]
+ },
+ {
+ "id": 48071,
+ "name": "Monogram Single Large Letter",
+ "required": 0,
+ "attributes": [
+ {
+ "id": 291208,
+ "name": "Style/Case",
+ "value": "Times Single Upper",
+ "isdefault": 0,
+ "sequence": 3,
+ "maxlength": 1
+ },
+ {
+ "id": 291209,
+ "name": "Style/Case",
+ "value": "Heirloom Single Upper",
+ "isdefault": 0,
+ "sequence": 3,
+ "maxlength": 1
+ },
+ {
+ "id": 291210,
+ "name": "Style/Case",
+ "value": "Athletic Single Upper",
+ "isdefault": 0,
+ "sequence": 3,
+ "maxlength": 1
+ },
+ {
+ "id": 291211,
+ "name": "Thread Color",
+ "value": "Black",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291212,
+ "name": "Thread Color",
+ "value": "Brown",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291213,
+ "name": "Thread Color",
+ "value": "Burgundy",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291214,
+ "name": "Thread Color",
+ "value": "Burlap",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291215,
+ "name": "Thread Color",
+ "value": "Cobalt",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291216,
+ "name": "Thread Color",
+ "value": "Copper",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291217,
+ "name": "Thread Color",
+ "value": "Cypress",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291218,
+ "name": "Thread Color",
+ "value": "Dark Brown",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291219,
+ "name": "Thread Color",
+ "value": "Dark Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291220,
+ "name": "Thread Color",
+ "value": "Dark Khaki",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291221,
+ "name": "Thread Color",
+ "value": "Dark Pink",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291222,
+ "name": "Thread Color",
+ "value": "Dark Teal",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291223,
+ "name": "Thread Color",
+ "value": "Eggplant",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291224,
+ "name": "Thread Color",
+ "value": "Emerald",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291225,
+ "name": "Thread Color",
+ "value": "Gold",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291226,
+ "name": "Thread Color",
+ "value": "Hunter Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291227,
+ "name": "Thread Color",
+ "value": "Light Purple",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291228,
+ "name": "Thread Color",
+ "value": "Lime",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291229,
+ "name": "Thread Color",
+ "value": "Loden",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291230,
+ "name": "Thread Color",
+ "value": "Lucerne Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291231,
+ "name": "Thread Color",
+ "value": "Mid Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291232,
+ "name": "Thread Color",
+ "value": "Mimosa Yellow",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291233,
+ "name": "Thread Color",
+ "value": "Natural",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291234,
+ "name": "Thread Color",
+ "value": "Navy",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291235,
+ "name": "Thread Color",
+ "value": "Nickel",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291236,
+ "name": "Thread Color",
+ "value": "Nile Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291237,
+ "name": "Thread Color",
+ "value": "Olive",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291238,
+ "name": "Thread Color",
+ "value": "Orange",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291239,
+ "name": "Thread Color",
+ "value": "Pink",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291240,
+ "name": "Thread Color",
+ "value": "Platinum",
+ "isdefault": 1,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291241,
+ "name": "Thread Color",
+ "value": "Purple",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291242,
+ "name": "Thread Color",
+ "value": "Rancho Rose",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291243,
+ "name": "Thread Color",
+ "value": "Red",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291244,
+ "name": "Thread Color",
+ "value": "Rose",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291245,
+ "name": "Thread Color",
+ "value": "Royal Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291246,
+ "name": "Thread Color",
+ "value": "Sage",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291247,
+ "name": "Thread Color",
+ "value": "Salmon",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291248,
+ "name": "Thread Color",
+ "value": "Sea Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291249,
+ "name": "Thread Color",
+ "value": "Standard Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291250,
+ "name": "Thread Color",
+ "value": "Taupe",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291251,
+ "name": "Thread Color",
+ "value": "Teal",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291252,
+ "name": "Thread Color",
+ "value": "Terra Cotta",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291253,
+ "name": "Thread Color",
+ "value": "Turkey Red",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291254,
+ "name": "Thread Color",
+ "value": "Walnut",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291255,
+ "name": "Thread Color",
+ "value": "White",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291256,
+ "name": "Monogram Single Line 1",
+ "value": "text",
+ "isdefault": 0,
+ "sequence": 1,
+ "maxlength": 1
+ },
+ {
+ "id": 353933,
+ "name": "Thread Color",
+ "value": "Isle Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 368999,
+ "name": "Thread Color",
+ "value": "Plum",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ }
+ ]
+ },
+ {
+ "id": 48072,
+ "name": "Monogram 3-Letter",
+ "required": 0,
+ "attributes": [
+ {
+ "id": 291257,
+ "name": "Style/Case",
+ "value": "Block Stack",
+ "isdefault": 0,
+ "sequence": 3,
+ "maxlength": 1
+ },
+ {
+ "id": 291258,
+ "name": "Style/Case",
+ "value": "Flared",
+ "isdefault": 0,
+ "sequence": 3,
+ "maxlength": 3
+ },
+ {
+ "id": 291259,
+ "name": "Style/Case",
+ "value": "Classic",
+ "isdefault": 0,
+ "sequence": 3,
+ "maxlength": 3
+ },
+ {
+ "id": 291260,
+ "name": "Style/Case",
+ "value": "Insignia",
+ "isdefault": 0,
+ "sequence": 3,
+ "maxlength": 3
+ },
+ {
+ "id": 291261,
+ "name": "Thread Color",
+ "value": "Black",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291262,
+ "name": "Thread Color",
+ "value": "Brown",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291263,
+ "name": "Thread Color",
+ "value": "Burgundy",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291264,
+ "name": "Thread Color",
+ "value": "Burlap",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291265,
+ "name": "Thread Color",
+ "value": "Cobalt",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291266,
+ "name": "Thread Color",
+ "value": "Copper",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291267,
+ "name": "Thread Color",
+ "value": "Cypress",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291268,
+ "name": "Thread Color",
+ "value": "Dark Brown",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291269,
+ "name": "Thread Color",
+ "value": "Dark Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291270,
+ "name": "Thread Color",
+ "value": "Dark Khaki",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291271,
+ "name": "Thread Color",
+ "value": "Dark Pink",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291272,
+ "name": "Thread Color",
+ "value": "Dark Teal",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291273,
+ "name": "Thread Color",
+ "value": "Eggplant",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291274,
+ "name": "Thread Color",
+ "value": "Emerald",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291275,
+ "name": "Thread Color",
+ "value": "Gold",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291276,
+ "name": "Thread Color",
+ "value": "Hunter Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291277,
+ "name": "Thread Color",
+ "value": "Light Purple",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291278,
+ "name": "Thread Color",
+ "value": "Lime",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291279,
+ "name": "Thread Color",
+ "value": "Loden",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291280,
+ "name": "Thread Color",
+ "value": "Lucerne Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291281,
+ "name": "Thread Color",
+ "value": "Mid Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291282,
+ "name": "Thread Color",
+ "value": "Mimosa Yellow",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291283,
+ "name": "Thread Color",
+ "value": "Natural",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291284,
+ "name": "Thread Color",
+ "value": "Navy",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291285,
+ "name": "Thread Color",
+ "value": "Nickel",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291286,
+ "name": "Thread Color",
+ "value": "Nile Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291287,
+ "name": "Thread Color",
+ "value": "Olive",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291288,
+ "name": "Thread Color",
+ "value": "Orange",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291289,
+ "name": "Thread Color",
+ "value": "Pink",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291290,
+ "name": "Thread Color",
+ "value": "Platinum",
+ "isdefault": 1,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291291,
+ "name": "Thread Color",
+ "value": "Purple",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291292,
+ "name": "Thread Color",
+ "value": "Rancho Rose",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291293,
+ "name": "Thread Color",
+ "value": "Red",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291294,
+ "name": "Thread Color",
+ "value": "Rose",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291295,
+ "name": "Thread Color",
+ "value": "Royal Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291296,
+ "name": "Thread Color",
+ "value": "Sage",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291297,
+ "name": "Thread Color",
+ "value": "Salmon",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291298,
+ "name": "Thread Color",
+ "value": "Sea Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291299,
+ "name": "Thread Color",
+ "value": "Standard Blue",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291300,
+ "name": "Thread Color",
+ "value": "Taupe",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291301,
+ "name": "Thread Color",
+ "value": "Teal",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291302,
+ "name": "Thread Color",
+ "value": "Terra Cotta",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291303,
+ "name": "Thread Color",
+ "value": "Turkey Red",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291304,
+ "name": "Thread Color",
+ "value": "Walnut",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291305,
+ "name": "Thread Color",
+ "value": "White",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 291306,
+ "name": "Monogram 3-Letter Line 1",
+ "value": "text",
+ "isdefault": 0,
+ "sequence": 1,
+ "maxlength": 3
+ },
+ {
+ "id": 353934,
+ "name": "Thread Color",
+ "value": "Isle Green",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ },
+ {
+ "id": 369000,
+ "name": "Thread Color",
+ "value": "Plum",
+ "isdefault": 0,
+ "sequence": 4,
+ "maxlength": 0
+ }
+ ]
+ },
+ {
+ "id": 3916,
+ "name": "Gift Boxing",
+ "required": 0,
+ "attributes": [
+ {
+ "id": 317426,
+ "name": "Box Style",
+ "value": "Tan/Green Ribbon",
+ "isdefault": 1,
+ "sequence": 1,
+ "maxlength": 0
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bc0a9005/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
index 9de546d..4eb09ab 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
@@ -77,7 +77,7 @@ public class EntityHelper {
* Verify that all fields in the expected are present in the returned, and have the same values
* via .equals. Does not recurse on object values. Also verifies there are no additional fields
* in the returned entity
- *
+ * ø
* @param expected
* @param returned
*/
[04/19] incubator-usergrid git commit: Removed duplicate MigrationRule
Posted by sf...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
index 73f1b6d..e2e6876 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
@@ -21,14 +21,28 @@ package org.apache.usergrid.persistence.collection.guice;
import org.apache.usergrid.persistence.core.guice.CommonModule;
+import org.apache.usergrid.persistence.core.guice.MaxMigrationModule;
+import org.apache.usergrid.persistence.core.guice.MaxMigrationVersion;
import org.apache.usergrid.persistence.core.guice.TestModule;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+
+import com.google.inject.multibindings.Multibinder;
public class TestCollectionModule extends TestModule {
@Override
protected void configure() {
+ /**
+ * Runtime modules
+ */
install( new CommonModule() );
install( new CollectionModule() );
+
+ /**
+ * Test modules
+ */
+ install(new MaxMigrationModule());
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/UniqueValueSerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/UniqueValueSerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/UniqueValueSerializationStrategyImplTest.java
index 9323520..2ad4e70 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/UniqueValueSerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/UniqueValueSerializationStrategyImplTest.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.collection.mvcc.stage.write;
import java.util.Collections;
import java.util.UUID;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.junit.Assert;
import org.junit.Rule;
@@ -30,7 +31,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index b18a957..8e14c4f 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@ -23,7 +23,7 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.stage.TestEntityGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
index 2830867..4ebfd24 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
@@ -24,7 +24,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index e3c4cd4..4b5ea7a 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -20,35 +20,28 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.UUID;
-import junit.framework.Assert;
-import org.apache.usergrid.persistence.core.test.UseModules;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.runner.RunWith;
-//import org.safehaus.chop.api.IterationChop;
-import org.safehaus.guicyfig.Env;
-import org.safehaus.guicyfig.Option;
-import org.safehaus.guicyfig.Overrides;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
-import org.apache.usergrid.persistence.core.astyanax.AstyanaxKeyspaceProvider;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
-import org.apache.usergrid.persistence.core.test.ITRunner;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationManagerFig;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.ArrayField;
import org.apache.usergrid.persistence.model.field.BooleanField;
import org.apache.usergrid.persistence.model.field.DoubleField;
import org.apache.usergrid.persistence.model.field.Field;
@@ -62,29 +55,26 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import junit.framework.Assert;
+
import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertFalse;
import static junit.framework.TestCase.assertNotNull;
import static junit.framework.TestCase.assertNull;
import static junit.framework.TestCase.assertTrue;
-import org.apache.usergrid.persistence.model.field.ArrayField;
import static org.mockito.Mockito.mock;
+//import org.safehaus.chop.api.IterationChop;
-/** @author tnine */
-//@IterationChop( iterations = 1000, threads = 2 )
-@RunWith( ITRunner.class )
-@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyImplTest {
- /** Our RX I/O threads and this should have the same value */
- private static final String CONNECTION_COUNT = "20";
+
+/**
+ * Tests for serialization strategy
+ */
+public abstract class MvccEntitySerializationStrategyImplTest {
- @Inject
private MvccEntitySerializationStrategy serializationStrategy;
- @Inject
- AstyanaxKeyspaceProvider provider;
@Inject
@Rule
@@ -94,16 +84,10 @@ public class MvccEntitySerializationStrategyImplTest {
public CassandraFig cassandraFig;
- @Inject
- public SerializationFig serializationFig;
-
- @Inject
- public MigrationManagerFig migrationManagerFig;
-
-
@Before
public void setup() {
assertNotNull( cassandraFig );
+ serializationStrategy = getMvccEntitySerializationStrategy();
}
@@ -684,5 +668,11 @@ public class MvccEntitySerializationStrategyImplTest {
}
+ /**
+ * Get the serialization strategy to test
+ * @return
+ */
+ protected abstract MvccEntitySerializationStrategy getMvccEntitySerializationStrategy();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
new file mode 100644
index 0000000..331bea2
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.PreviousImpl;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+
+import com.google.inject.Inject;
+
+import static org.junit.Assert.assertTrue;
+
+
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class MvccEntitySerializationStrategyProxyV1Test extends MvccEntitySerializationStrategyImplTest {
+
+ @Inject
+ @ProxyImpl
+ private MvccEntitySerializationStrategy serializationStrategy;
+
+ @Override
+ protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
+ return serializationStrategy;
+ }
+
+ @Inject
+ protected MigrationInfoSerialization migrationInfoSerialization;
+
+ private int existingVersion;
+
+
+ /**
+ * We need to run our migration to ensure that we are on the current version, and everything still functions
+ * correctly
+ */
+ @Before
+ public void setMigrationVersion() {
+ existingVersion = migrationInfoSerialization.getVersion();
+
+ //set our version to 0 so it uses both impls of the proxy
+ migrationInfoSerialization.setVersion( MvccEntitySerializationStrategyProxyImpl.MIGRATION_VERSION-1 );
+ }
+
+
+ @After
+ public void reSetMigrationVersion() {
+ migrationInfoSerialization.setVersion( existingVersion );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
new file mode 100644
index 0000000..19b2cae
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+
+import com.google.inject.Inject;
+
+
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerializationStrategyImplTest {
+
+ @Inject
+ @ProxyImpl
+ private MvccEntitySerializationStrategy serializationStrategy;
+
+
+ @Override
+ protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
+ return serializationStrategy;
+ }
+
+
+ @Inject
+ protected MigrationInfoSerialization migrationInfoSerialization;
+
+ private int existingVersion;
+
+
+ /**
+ * We need to run our migration to ensure that we are on the current version, and everything still functions
+ * correctly
+ */
+ @Before
+ public void setMigrationVersion() {
+ existingVersion = migrationInfoSerialization.getVersion();
+
+ //set our new version, so that is will run through the new code
+ migrationInfoSerialization.setVersion( MvccEntitySerializationStrategyProxyImpl.MIGRATION_VERSION );
+ }
+
+
+ @After
+ public void reSetMigrationVersion() {
+ migrationInfoSerialization.setVersion( existingVersion );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
new file mode 100644
index 0000000..01d9a9b
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.PreviousImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+
+import com.google.inject.Inject;
+
+
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class MvccEntitySerializationStrategyV1ImplTest extends MvccEntitySerializationStrategyImplTest {
+
+ @Inject
+ @PreviousImpl
+ private MvccEntitySerializationStrategy serializationStrategy;
+
+ @Override
+ protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
+ return serializationStrategy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
new file mode 100644
index 0000000..260bcf2
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import org.junit.runner.RunWith;
+
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.PreviousImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+
+import com.google.inject.Inject;
+
+
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySerializationStrategyImplTest {
+
+ @Inject
+ @CurrentImpl
+ private MvccEntitySerializationStrategy serializationStrategy;
+
+ @Override
+ protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
+ return serializationStrategy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
index a9cf3a0..4f96594 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
@@ -34,7 +34,7 @@ import org.safehaus.guicyfig.Option;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
index fb0794f..f7cc615 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
@@ -30,7 +30,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/log4j.properties b/stack/corepersistence/collection/src/test/resources/log4j.properties
index e765c29..53714a8 100644
--- a/stack/corepersistence/collection/src/test/resources/log4j.properties
+++ b/stack/corepersistence/collection/src/test/resources/log4j.properties
@@ -1,3 +1,4 @@
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -8,24 +9,28 @@
#
# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
# suppress inspection "UnusedProperty" for whole file
-log4j.rootLogger=INFO
+log4j.rootLogger=INFO,stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %p %c{3}.%M(%L)<%t>- %m%n
-#log4j.logger.org.safehaus.chop.plugin=DEBUG
+log4j.logger.org.safehaus.chop.plugin=DEBUG
log4j.logger.org.safehaus.guicyfig=ERROR
log4j.logger.org.safehaus.chop.api.store.amazon=DEBUG
log4j.logger.org.apache.http=ERROR
log4j.logger.com.amazonaws.request=ERROR
log4j.logger.cassandra.db=ERROR
-#log4j.logger.org.apache.usergrid.persistence.collection=DEBUG
+#log4j.logger.org.apache.usergrid=DEBUG
+log4j.logger.org.apache.usergrid.persistence.collection=TRACE
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationModule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationModule.java
new file mode 100644
index 0000000..2d91492
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationModule.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.guice;
+
+
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.multibindings.Multibinder;
+
+
+/**
+ * Install this module in your tests if you want the max version to always be set
+ * this ensures that the system will be on Integer.MAX version, there for
+ */
+public class MaxMigrationModule extends AbstractModule {
+ @Override
+ protected void configure() {
+ Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
+ dataMigrationMultibinder.addBinding().to( MaxMigrationVersion.class );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
new file mode 100644
index 0000000..be45ab5
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MaxMigrationVersion.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.guice;
+
+
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+
+
+/**
+ * A simple migration that sets the version to max. This way our integration tests always test the latest code
+ */
+public class MaxMigrationVersion implements DataMigration {
+ @Override
+ public void migrate( final ProgressObserver observer ) throws Throwable {
+ //no op, just needs to run to be set
+ }
+
+
+ @Override
+ public int getVersion() {
+ return Integer.MAX_VALUE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java
index 522af61..cba88aa 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/guice/MigrationManagerRule.java
@@ -5,6 +5,7 @@ import org.junit.rules.ExternalResource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
@@ -16,23 +17,33 @@ import com.google.inject.Singleton;
*/
@Singleton
public class MigrationManagerRule extends ExternalResource {
- private static final Logger LOG = LoggerFactory.getLogger( MigrationManagerRule.class );
+ private static final Logger logger = LoggerFactory.getLogger( MigrationManagerRule.class );
+
private MigrationManager migrationManager;
+ private DataMigrationManager dataMigrationManager;
@Inject
- public void setMigrationManager( MigrationManager migrationManager ) {
+ public void setMigrationManager( final MigrationManager migrationManager ) {
this.migrationManager = migrationManager;
}
+ @Inject
+ public void setDataMigrationManager(final DataMigrationManager dataMigrationManager){
+ this.dataMigrationManager = dataMigrationManager;
+ }
@Override
protected void before() throws MigrationException {
- LOG.info( "Starting migration" );
+ logger.info( "Starting migration" );
migrationManager.migrate();
- LOG.info( "Migration complete" );
+ logger.info("Migrating data");
+
+ dataMigrationManager.migrate();
+
+ logger.info( "Migration complete" );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index da92124..69e0422 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -26,7 +26,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
index 916bdaa..d3bf24e 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
@@ -39,7 +39,7 @@ import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.time.StopWatch;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
index dfda4b1..a9035d3 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
@@ -29,7 +29,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
index 7b912a0..cc131d1 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
@@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.time.StopWatch;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/SimpleTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/SimpleTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/SimpleTest.java
index b840c51..0a27a6b 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/SimpleTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/SimpleTest.java
@@ -8,7 +8,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
index 87a0f6c..20343bc 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.graph.guice;
import org.apache.usergrid.persistence.core.guice.CommonModule;
+import org.apache.usergrid.persistence.core.guice.MaxMigrationModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
@@ -30,7 +31,17 @@ public class TestGraphModule extends TestModule {
@Override
protected void configure() {
+ /**
+ * Runtime modules
+ */
install( new CommonModule());
install( new GraphModule() );
+
+
+ /**
+ * Test modules
+ */
+ install(new MaxMigrationModule());
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
index f5e9dc8..b8c5b3e 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
@@ -31,7 +31,7 @@ import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
index aa5ae5f..d91bfc3 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
@@ -30,7 +30,7 @@ import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
index d27aca7..cdb284d 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
@@ -29,7 +29,7 @@ import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
index 156d1d0..1b57c2d 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
@@ -29,7 +29,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java
index e166194..1ed1742 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetadataSerializationTest.java
@@ -27,7 +27,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
index 9dd362f..b1bcf2d 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
@@ -27,7 +27,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
index 69d0a20..34aea46 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
@@ -32,7 +32,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphFig;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
index 2391192..d74eb4e 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
@@ -27,7 +27,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
index cdac709..9a87bba 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
@@ -27,7 +27,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
index a443086..f80f44d 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
@@ -27,7 +27,7 @@ import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
index f588663..28000fe 100644
--- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
+++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
@@ -26,7 +26,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.map.guice.TestMapModule;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
[10/19] incubator-usergrid git commit: Renamed entity log load
methods to be more clear. Also updated documentation.
Posted by sf...@apache.org.
Renamed entity log load methods to be more clear. Also updated documentation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/262ad8ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/262ad8ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/262ad8ba
Branch: refs/heads/two-dot-o
Commit: 262ad8ba3a0ab9e743b6b0013725f1079d014d52
Parents: eb91d13
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 20 10:08:53 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 20 10:08:53 2014 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityDeleteListener.java | 6 ++---
.../CpEntityDeleteListenerTest.java | 4 ++--
.../mvcc/MvccEntitySerializationStrategy.java | 18 +++++++-------
.../mvcc/stage/delete/MarkCommit.java | 2 +-
.../serialization/impl/EntityRepairImpl.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 23 +++++++-----------
...vccEntitySerializationStrategyProxyImpl.java | 16 ++++++-------
.../serialization/EntityRepairImplTest.java | 2 +-
...ccEntitySerializationSTrategyV2FixTests.java | 2 +-
...MvccEntitySerializationStrategyImplTest.java | 25 ++++++++++++--------
10 files changed, 50 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
index 8785ab2..e75756f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
@@ -69,10 +69,8 @@ public class CpEntityDeleteListener {
return Observable.create( new ObservableIterator<MvccEntity>( "deleteEntities" ) {
@Override
protected Iterator<MvccEntity> getIterator() {
- Iterator<MvccEntity> iterator = entityMetadataSerialization.loadHistory(
- entityEvent.getCollectionScope(),
- entity.getId(),
- entity.getVersion(),
+ Iterator<MvccEntity> iterator = entityMetadataSerialization.loadAscendingHistory(
+ entityEvent.getCollectionScope(), entity.getId(), entity.getVersion(),
serializationFig.getHistorySize() );
return iterator;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
index 6efab38..2147c45 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
@@ -23,7 +23,6 @@ import java.util.UUID;
import org.junit.Before;
import org.junit.Test;
-import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
@@ -81,7 +80,8 @@ public class CpEntityDeleteListenerTest {
ArrayList<MvccEntity> entityList = new ArrayList<>();
entityList.add(entity);
when(mvccEntitySerializationStrategy.delete(scope, entityId, id)).thenReturn(batch);
- when(mvccEntitySerializationStrategy.loadHistory(scope, entityId, id, serializationFig.getHistorySize())).thenReturn(entityList.iterator());
+ when(mvccEntitySerializationStrategy.loadAscendingHistory( scope, entityId, id,
+ serializationFig.getHistorySize() )).thenReturn(entityList.iterator());
Observable<EntityVersion> observable = listener.receive(entityEvent);
EntityVersion entityEventReturned = observable.toBlocking().last();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
index 3c694c5..8a13115 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
@@ -62,24 +62,26 @@ public interface MvccEntitySerializationStrategy extends Migration {
* @param context The context to persist the entity into
* @param entityId The entity id to load
* @param version The max version to seek from. I.E a stored version <= this argument
- * @param fetchSize The maximum size to return. If you receive this size, there may be more versions to load.
+ * @param fetchSize The fetch size to return for each trip to cassandra.
*
- * @return A list of entities up to max size ordered from max(UUID)=> min(UUID). The return value should be null
+ * @return An iterator of entities ordered from max(UUID)=> min(UUID). The return value should be null
* safe and return an empty list when there are no matches
*/
- public Iterator<MvccEntity> load( CollectionScope context, Id entityId, UUID version, int fetchSize );
+ public Iterator<MvccEntity> loadDescendingHistory( CollectionScope context, Id entityId, UUID version,
+ int fetchSize );
/**
- * Load a historical list of entities, from highest to lowest of the entity with versions < version up to maxSize elements
+ * Load a historical list of entities, from lowest to highest entity with versions < version up to maxSize elements
*
* @param context The context to persist the entity into
* @param entityId The entity id to load
- * @param version The max version to seek from. I.E a stored version < this argument
- * @param fetchSize The maximum size to return. If you receive this size, there may be more versions to load.
- * @return A list of entities up to max size ordered from max(UUID)=> min(UUID). The return value should be null
+ * @param version The max version to seek to. I.E a stored version < this argument
+ * @param fetchSize The fetch size to return for each trip to cassandra.
+ * @return An iterator of entities ordered from min(UUID)=> max(UUID). The return value should be null
* safe and return an empty list when there are no matches
*/
- public Iterator<MvccEntity> loadHistory( CollectionScope context, Id entityId, UUID version, int fetchSize );
+ public Iterator<MvccEntity> loadAscendingHistory( CollectionScope context, Id entityId, UUID version,
+ int fetchSize );
/**
* Mark this this version as deleted from the persistence store, but keep the version to mark that is has been cleared This
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 5bcb9f8..e31aba5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -129,7 +129,7 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
@Override
protected Iterator<MvccEntity> getIterator() {
Iterator<MvccEntity> entities =
- entityStrat.load( collectionScope, entityId, entity.getVersion(), 100 );
+ entityStrat.loadDescendingHistory( collectionScope, entityId, entity.getVersion(), 100 );
return entities;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
index a94c688..4226fe6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
@@ -69,7 +69,7 @@ public class EntityRepairImpl implements EntityRepair {
partialEntities.add( targetEntity );
final Iterator<MvccEntity> results = mvccEntitySerializationStrategy
- .load( collectionScope, targetEntity.getId(), targetEntity.getVersion(),
+ .loadDescendingHistory( collectionScope, targetEntity.getId(), targetEntity.getVersion(),
serializationFig.getBufferSize() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index 1ec027f..4138f8d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -48,7 +48,6 @@ import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -64,8 +63,6 @@ import com.netflix.astyanax.model.ColumnList;
import com.netflix.astyanax.model.Row;
import com.netflix.astyanax.query.RowQuery;
import com.netflix.astyanax.serializers.AbstractSerializer;
-import com.netflix.astyanax.serializers.ByteBufferSerializer;
-import com.netflix.astyanax.serializers.BytesArraySerializer;
import com.netflix.astyanax.serializers.UUIDSerializer;
@@ -77,7 +74,6 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
private static final Logger log = LoggerFactory.getLogger( MvccLogEntrySerializationStrategyImpl.class );
-
private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
@@ -196,7 +192,8 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
final Column<UUID> column = columns.getColumnByIndex( 0 );
- final MvccEntity parsedEntity = new MvccColumnParser( entityId, entityJsonSerializer ).parseColumn( column );
+ final MvccEntity parsedEntity =
+ new MvccColumnParser( entityId, entityJsonSerializer ).parseColumn( column );
//we *might* need to repair, it's not clear so check before loading into result sets
final MvccEntity maybeRepaired = repair.maybeRepair( collectionScope, parsedEntity );
@@ -209,8 +206,8 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
@Override
- public Iterator<MvccEntity> load( final CollectionScope collectionScope, final Id entityId, final UUID version,
- final int fetchSize ) {
+ public Iterator<MvccEntity> loadDescendingHistory( final CollectionScope collectionScope, final Id entityId,
+ final UUID version, final int fetchSize ) {
Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
Preconditions.checkNotNull( entityId, "entity id is required" );
@@ -239,8 +236,8 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
@Override
- public Iterator<MvccEntity> loadHistory( final CollectionScope collectionScope, final Id entityId,
- final UUID version, final int fetchSize ) {
+ public Iterator<MvccEntity> loadAscendingHistory( final CollectionScope collectionScope, final Id entityId,
+ final UUID version, final int fetchSize ) {
Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
Preconditions.checkNotNull( entityId, "entity id is required" );
@@ -393,10 +390,9 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
deSerialized = column.getValue( entityJsonSerializer );
}
catch ( DataCorruptionException e ) {
- log.error(
- "DATA CORRUPTION DETECTED when de-serializing entity with Id {} and version {}. This means the"
- + " write was truncated.",
- id, version );
+ log.error(
+ "DATA CORRUPTION DETECTED when de-serializing entity with Id {} and version {}. This means the"
+ + " write was truncated.", id, version );
//return an empty entity, we can never load this one, and we don't want it to bring the system
//to a grinding halt
return new MvccEntityImpl( id, version, MvccEntity.Status.DELETED, Optional.<Entity>absent() );
@@ -414,7 +410,6 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
/**
* Return the entity serializer for this instance
- * @return
*/
protected abstract AbstractSerializer<EntityWrapper> getEntitySerializer();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
index a9e01b1..cf17812 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
@@ -96,24 +96,24 @@ public class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySeria
@Override
- public Iterator<MvccEntity> load( final CollectionScope context, final Id entityId, final UUID version,
- final int fetchSize ) {
+ public Iterator<MvccEntity> loadDescendingHistory( final CollectionScope context, final Id entityId,
+ final UUID version, final int fetchSize ) {
if ( isOldVersion() ) {
- return previous.load( context, entityId, version, fetchSize );
+ return previous.loadDescendingHistory( context, entityId, version, fetchSize );
}
- return current.load( context, entityId, version, fetchSize );
+ return current.loadDescendingHistory( context, entityId, version, fetchSize );
}
@Override
- public Iterator<MvccEntity> loadHistory( final CollectionScope context, final Id entityId, final UUID version,
- final int fetchSize ) {
+ public Iterator<MvccEntity> loadAscendingHistory( final CollectionScope context, final Id entityId,
+ final UUID version, final int fetchSize ) {
if ( isOldVersion() ) {
- return previous.loadHistory( context, entityId, version, fetchSize );
+ return previous.loadAscendingHistory( context, entityId, version, fetchSize );
}
- return current.loadHistory( context, entityId, version, fetchSize );
+ return current.loadAscendingHistory( context, entityId, version, fetchSize );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
index 4326e10..6194c80 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
@@ -101,7 +101,7 @@ public class EntityRepairImplTest {
//mock up returning
when( mvccEntitySerializationStrategy
- .loadHistory( scope, simpleId, v3.getVersion(), serializationFig.getBufferSize() ) )
+ .loadDescendingHistory( scope, simpleId, v3.getVersion(), serializationFig.getBufferSize() ) )
.thenReturn( Arrays.<MvccEntity>asList( v3, v2, v1 ).iterator() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
index 293922e..0d7e335 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
@@ -69,7 +69,7 @@ public abstract class MvccEntitySerializationSTrategyV2FixTests extends MvccEnti
//now load it
final Iterator<MvccEntity> loaded =
- getMvccEntitySerializationStrategy().loadHistory( context, entity.getId(), version, 100 );
+ getMvccEntitySerializationStrategy().loadAscendingHistory( context, entity.getId(), version, 100 );
assertTrue( loaded.hasNext() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/262ad8ba/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index 4b5ea7a..d54d958 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -402,7 +402,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
//now ask for up to 10 versions from the current version, we should get cleared, v2, v1
UUID current = UUIDGenerator.newTimeUUID();
- Iterator<MvccEntity> entities = serializationStrategy.load( context, id, current, 3 );
+ Iterator<MvccEntity> entities = serializationStrategy.loadDescendingHistory( context, id, current, 3 );
MvccEntity first = entities.next();
assertEquals( clearedV3, first);
@@ -422,7 +422,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
serializationStrategy.delete( context, id , version1 ).execute();
serializationStrategy.delete( context, id , version2 ).execute();
- entities = serializationStrategy.load( context, id, current, 3 );
+ entities = serializationStrategy.loadDescendingHistory( context, id, current, 3 );
first = entities.next();
assertEquals( clearedV3, first );
@@ -432,7 +432,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
serializationStrategy.delete( context, id , version3 ).execute();
- entities = serializationStrategy.load( context, id, current, 3 );
+ entities = serializationStrategy.loadDescendingHistory( context, id, current, 3 );
Assert.assertTrue( !entities.hasNext());
}
@@ -464,7 +464,8 @@ public abstract class MvccEntitySerializationStrategyImplTest {
MvccEntity savedV2 = new MvccEntityImpl(id, version2, MvccEntity.Status.COMPLETE, Optional.of(entityv2));
serializationStrategy.write(context, savedV2).execute();
- Iterator<MvccEntity> entities = serializationStrategy.loadHistory(context,id,savedV2.getVersion(),20);
+ Iterator<MvccEntity> entities = serializationStrategy.loadAscendingHistory( context, id, savedV2.getVersion(),
+ 20 );
assertTrue(entities.hasNext());
assertEquals(saved.getVersion(), entities.next().getVersion());
assertEquals(savedV2.getVersion(), entities.next().getVersion());
@@ -639,7 +640,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
@Test(expected = NullPointerException.class)
public void loadListParamContext() throws ConnectionException {
- serializationStrategy.load( null, new SimpleId( "test" ), UUIDGenerator.newTimeUUID(), 1 );
+ serializationStrategy.loadDescendingHistory( null, new SimpleId( "test" ), UUIDGenerator.newTimeUUID(), 1 );
}
@@ -647,8 +648,9 @@ public abstract class MvccEntitySerializationStrategyImplTest {
public void loadListParamEntityId() throws ConnectionException {
serializationStrategy
- .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), null, UUIDGenerator.newTimeUUID(),
- 1 );
+ .loadDescendingHistory(
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), null,
+ UUIDGenerator.newTimeUUID(), 1 );
}
@@ -656,15 +658,18 @@ public abstract class MvccEntitySerializationStrategyImplTest {
public void loadListParamVersion() throws ConnectionException {
serializationStrategy
- .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ), null, 1 );
+ .loadDescendingHistory(
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" ),
+ new SimpleId( "test" ), null, 1 );
}
@Test(expected = IllegalArgumentException.class)
public void loadListParamSize() throws ConnectionException {
- serializationStrategy.load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ),
- UUIDGenerator.newTimeUUID(), 0 );
+ serializationStrategy.loadDescendingHistory(
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" ),
+ new SimpleId( "test" ), UUIDGenerator.newTimeUUID(), 0 );
}
[06/19] incubator-usergrid git commit: Removed redundant load method
Posted by sf...@apache.org.
Removed redundant load method
Finished V2 tests and refactor. Issue proven in tests which fail.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8beab448
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8beab448
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8beab448
Branch: refs/heads/two-dot-o
Commit: 8beab4489672e403462ecdc9bfe6f3061954c076
Parents: 2bd1c95
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 19 17:37:44 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 19 17:37:44 2014 -0700
----------------------------------------------------------------------
.../mvcc/MvccEntitySerializationStrategy.java | 12 ---
.../mvcc/stage/delete/MarkCommit.java | 2 +-
.../serialization/impl/EntityRepairImpl.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 30 ------
...vccEntitySerializationStrategyProxyImpl.java | 9 --
.../collection/EntityCollectionManagerIT.java | 28 +----
.../serialization/EntityRepairImplTest.java | 2 +-
...ccEntitySerializationSTrategyV2FixTests.java | 82 ++++++++++++++
...MvccEntitySerializationStrategyImplTest.java | 17 +--
...cEntitySerializationStrategyProxyV2Test.java | 2 +-
...ccEntitySerializationStrategyV2ImplTest.java | 4 +-
.../collection/util/EntityHelper.java | 106 +++++++++++++++++++
12 files changed, 208 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
index 93288af..faa76a3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
@@ -56,18 +56,6 @@ public interface MvccEntitySerializationStrategy extends Migration {
*/
public EntitySet load( CollectionScope scope, Collection<Id> entityIds, UUID maxVersion);
- /**
- * Load a list, from highest to lowest of the entity with versions <= version up to maxSize elements
- *
- * @param context The context to persist the entity into
- * @param entityId The entity id to load
- * @param version The max version to seek from. I.E a stored version >= this argument
- * @param fetchSize The maximum size to return. If you receive this size, there may be more versions to load.
- *
- * @return A list of entities up to max size ordered from max(UUID)=> min(UUID). The return value should be null
- * safe and return an empty list when there are no matches
- */
- public Iterator<MvccEntity> load( CollectionScope context, Id entityId, UUID version, int fetchSize );
/**
* Load a historical list of entities, from highest to lowest of the entity with versions <= version up to maxSize elements
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 5bcb9f8..fab904d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -129,7 +129,7 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
@Override
protected Iterator<MvccEntity> getIterator() {
Iterator<MvccEntity> entities =
- entityStrat.load( collectionScope, entityId, entity.getVersion(), 100 );
+ entityStrat.loadHistory( collectionScope, entityId, entity.getVersion(), 100 );
return entities;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
index a94c688..a01ffdb 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
@@ -69,7 +69,7 @@ public class EntityRepairImpl implements EntityRepair {
partialEntities.add( targetEntity );
final Iterator<MvccEntity> results = mvccEntitySerializationStrategy
- .load( collectionScope, targetEntity.getId(), targetEntity.getVersion(),
+ .loadHistory( collectionScope, targetEntity.getId(), targetEntity.getVersion(),
serializationFig.getBufferSize() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index 1ec027f..c964b68 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -209,36 +209,6 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
@Override
- public Iterator<MvccEntity> load( final CollectionScope collectionScope, final Id entityId, final UUID version,
- final int fetchSize ) {
-
- Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
- Preconditions.checkNotNull( entityId, "entity id is required" );
- Preconditions.checkNotNull( version, "version is required" );
- Preconditions.checkArgument( fetchSize > 0, "max Size must be greater than 0" );
-
-
- final Id applicationId = collectionScope.getApplication();
- final Id ownerId = collectionScope.getOwner();
- final String collectionName = collectionScope.getName();
-
- final CollectionPrefixedKey<Id> collectionPrefixedKey =
- new CollectionPrefixedKey<>( collectionName, ownerId, entityId );
-
-
- final ScopedRowKey<CollectionPrefixedKey<Id>> rowKey =
- ScopedRowKey.fromKey( applicationId, collectionPrefixedKey );
-
-
- RowQuery<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> query =
- keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( rowKey )
- .withColumnRange( version, null, false, fetchSize );
-
- return new ColumnNameIterator( query, new MvccColumnParser( entityId, entityJsonSerializer ), false );
- }
-
-
- @Override
public Iterator<MvccEntity> loadHistory( final CollectionScope collectionScope, final Id entityId,
final UUID version, final int fetchSize ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
index a9e01b1..f5cf642 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
@@ -95,15 +95,6 @@ public class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySeria
}
- @Override
- public Iterator<MvccEntity> load( final CollectionScope context, final Id entityId, final UUID version,
- final int fetchSize ) {
- if ( isOldVersion() ) {
- return previous.load( context, entityId, version, fetchSize );
- }
-
- return current.load( context, entityId, version, fetchSize );
- }
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 2d18675..db5e3a4 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -28,6 +28,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
@@ -43,7 +44,6 @@ import org.apache.usergrid.persistence.model.field.BooleanField;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.field.IntegerField;
import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.fasterxml.uuid.UUIDComparator;
import com.google.inject.Inject;
@@ -713,29 +713,8 @@ public class EntityCollectionManagerIT {
public void largeEntityWriteRead() {
final int setSize = 65535 * 2;
- int currentLength = 0;
-
- final Entity entity = new Entity( new SimpleId( "test" ) );
-
- //generate a really large string value
- StringBuilder builder = new StringBuilder();
-
- for ( int i = 0; i < 100; i++ ) {
- builder.append( UUIDGenerator.newTimeUUID().toString() );
- }
-
- final String value = builder.toString();
-
-
- //loop until our size is beyond the set size
- for ( int i = 0; currentLength < setSize; i++ ) {
- final String key = "newStringField" + i;
-
- entity.setField( new StringField( key, value ) );
-
- currentLength += key.length() + value.length();
- }
+ final Entity entity = EntityHelper.generateEntity( setSize );
//now we have one massive, entity, save it and retrieve it.
CollectionScope context =
@@ -751,6 +730,7 @@ public class EntityCollectionManagerIT {
//now load it
final Entity loaded = manager.load( entity.getId() ).toBlocking().last();
- assertEquals( entity, loaded );
+
+ EntityHelper.verifySame( entity, loaded );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
index 27f39dc..4326e10 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
@@ -101,7 +101,7 @@ public class EntityRepairImplTest {
//mock up returning
when( mvccEntitySerializationStrategy
- .load( scope, simpleId, v3.getVersion(), serializationFig.getBufferSize() ) )
+ .loadHistory( scope, simpleId, v3.getVersion(), serializationFig.getBufferSize() ) )
.thenReturn( Arrays.<MvccEntity>asList( v3, v2, v1 ).iterator() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
new file mode 100644
index 0000000..293922e
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public abstract class MvccEntitySerializationSTrategyV2FixTests extends MvccEntitySerializationStrategyImplTest {
+
+
+ /**
+ * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
+ */
+ @Test
+ public void largeEntityWriteRead() throws ConnectionException {
+ final int setSize = 65535 * 2;
+
+ final Entity entity = EntityHelper.generateEntity( setSize );
+
+ //now we have one massive, entity, save it and retrieve it.
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+
+
+ final Id simpleId = new SimpleId( "test" );
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
+
+ final MvccEntity mvccEntity = new MvccEntityImpl( simpleId, version, status, entity );
+
+
+ getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
+
+ //now load it
+ final Iterator<MvccEntity> loaded =
+ getMvccEntitySerializationStrategy().loadHistory( context, entity.getId(), version, 100 );
+
+ assertTrue( loaded.hasNext() );
+
+ final MvccEntity loadedEntity = loaded.next();
+
+ assertEquals( "The loaded entity should match the stored entity", mvccEntity, loadedEntity );
+
+ EntityHelper.verifySame( entity, loadedEntity.getEntity().get() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index 4b5ea7a..bc7c783 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -402,7 +402,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
//now ask for up to 10 versions from the current version, we should get cleared, v2, v1
UUID current = UUIDGenerator.newTimeUUID();
- Iterator<MvccEntity> entities = serializationStrategy.load( context, id, current, 3 );
+ Iterator<MvccEntity> entities = serializationStrategy.loadHistory( context, id, current, 3 );
MvccEntity first = entities.next();
assertEquals( clearedV3, first);
@@ -422,7 +422,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
serializationStrategy.delete( context, id , version1 ).execute();
serializationStrategy.delete( context, id , version2 ).execute();
- entities = serializationStrategy.load( context, id, current, 3 );
+ entities = serializationStrategy.loadHistory( context, id, current, 3 );
first = entities.next();
assertEquals( clearedV3, first );
@@ -432,7 +432,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
serializationStrategy.delete( context, id , version3 ).execute();
- entities = serializationStrategy.load( context, id, current, 3 );
+ entities = serializationStrategy.loadHistory( context, id, current, 3 );
Assert.assertTrue( !entities.hasNext());
}
@@ -639,7 +639,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
@Test(expected = NullPointerException.class)
public void loadListParamContext() throws ConnectionException {
- serializationStrategy.load( null, new SimpleId( "test" ), UUIDGenerator.newTimeUUID(), 1 );
+ serializationStrategy.loadHistory( null, new SimpleId( "test" ), UUIDGenerator.newTimeUUID(), 1 );
}
@@ -647,8 +647,8 @@ public abstract class MvccEntitySerializationStrategyImplTest {
public void loadListParamEntityId() throws ConnectionException {
serializationStrategy
- .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), null, UUIDGenerator.newTimeUUID(),
- 1 );
+ .loadHistory( new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" ),
+ null, UUIDGenerator.newTimeUUID(), 1 );
}
@@ -656,14 +656,15 @@ public abstract class MvccEntitySerializationStrategyImplTest {
public void loadListParamVersion() throws ConnectionException {
serializationStrategy
- .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ), null, 1 );
+ .loadHistory( new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" ),
+ new SimpleId( "test" ), null, 1 );
}
@Test(expected = IllegalArgumentException.class)
public void loadListParamSize() throws ConnectionException {
- serializationStrategy.load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ),
+ serializationStrategy.loadHistory( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ),
UUIDGenerator.newTimeUUID(), 0 );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
index 19b2cae..02eccb1 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
@@ -36,7 +36,7 @@ import com.google.inject.Inject;
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerializationStrategyImplTest {
+public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerializationSTrategyV2FixTests {
@Inject
@ProxyImpl
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
index 260bcf2..012eeff 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
@@ -34,7 +34,7 @@ import com.google.inject.Inject;
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySerializationStrategyImplTest {
+public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySerializationSTrategyV2FixTests {
@Inject
@CurrentImpl
@@ -44,4 +44,6 @@ public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySeriali
protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
return serializationStrategy;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8beab448/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
new file mode 100644
index 0000000..9de546d
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.util;
+
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+
+/**
+ * Utilities for generating test data.
+ */
+public class EntityHelper {
+
+ /**
+ * Generates an entity with all string fields to have at least the minSize
+ * number of characters in the field names + field values
+ * @param minSize
+ * @return
+ */
+ public static Entity generateEntity(final int minSize){
+ int currentLength = 0;
+
+ final Entity entity = new Entity( new SimpleId( "test" ) );
+
+ //generate a really large string value
+ StringBuilder builder = new StringBuilder();
+
+ for ( int i = 0; i < 100; i++ ) {
+ builder.append( UUIDGenerator.newTimeUUID().toString() );
+ }
+
+ final String value = builder.toString();
+
+
+ //loop until our size is beyond the set size
+ for ( int i = 0; currentLength < minSize; i++ ) {
+ final String key = "newStringField" + i;
+
+ entity.setField( new StringField( key, value ) );
+
+ currentLength += key.length() + value.length();
+ }
+
+ return entity;
+ }
+
+
+ /**
+ * Verify that all fields in the expected are present in the returned, and have the same values
+ * via .equals. Does not recurse on object values. Also verifies there are no additional fields
+ * in the returned entity
+ *
+ * @param expected
+ * @param returned
+ */
+ public static void verifySame(final Entity expected, final Entity returned){
+
+ //perform object equals
+ assertEquals("Expected same entity equality", expected, returned);
+
+ final Collection<Field> expectedFields = expected.getFields();
+
+ final Map<String, Field> returnedFields = new HashMap<>(returned.getFieldMap());
+
+ for(Field expectedField: expectedFields){
+
+ final Field returnedField = returnedFields.get( expectedField.getName() );
+
+ assertNotNull("Field " + expectedField.getName() + " exists in returned entity", returnedField );
+
+ assertEquals("Field values should match", expectedField.getValue(), returnedField.getValue());
+
+ returnedFields.remove( expectedField.getName() );
+ }
+
+ assertEquals("There are no additional fields in the returned entity", 0, returnedFields.size());
+ }
+}
[12/19] incubator-usergrid git commit: Refactored generation of
collection scope to be re-used with CpNamingUtils for consistency
Posted by sf...@apache.org.
Refactored generation of collection scope to be re-used with CpNamingUtils for consistency
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9ed79642
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9ed79642
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9ed79642
Branch: refs/heads/two-dot-o
Commit: 9ed796423f15a8ef33137b113f6e04c578994b21
Parents: 68f4e0f
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 20 14:15:24 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 20 14:23:38 2014 -0700
----------------------------------------------------------------------
.../migration/EntityDataMigration.java | 139 +++++++++++++++++
.../migration/EntityTypeMappingMigration.java | 28 ++--
.../migration/GraphShardVersionMigration.java | 104 +++++++------
.../rx/AllEntitiesInSystemObservable.java | 27 ++--
.../corepersistence/StaleIndexCleanupTest.java | 5 +-
.../migration/EntityTypeMappingMigrationIT.java | 74 +++++-----
.../migration/GraphShardVersionMigrationIT.java | 148 +++++++++++--------
.../rx/AllEntitiesInSystemObservableIT.java | 23 ++-
8 files changed, 364 insertions(+), 184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ed79642/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java
new file mode 100644
index 0000000..79d31a8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.PreviousImpl;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationException;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import rx.functions.Action1;
+
+
+/**
+ * Migration for migrating graph edges to the new Shards
+ */
+public class EntityDataMigration implements DataMigration {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( EntityDataMigration.class );
+
+
+ private final MvccEntitySerializationStrategy v1Serialization;
+ private final MvccEntitySerializationStrategy v2Serialization;
+
+ private final ManagerCache managerCache;
+ private final Keyspace keyspace;
+
+
+ @Inject
+ public EntityDataMigration( @PreviousImpl final MvccEntitySerializationStrategy v1Serialization,
+ @CurrentImpl final MvccEntitySerializationStrategy v2Serialization,
+ final ManagerCache managerCache, final Keyspace keyspace ) {
+ this.v1Serialization = v1Serialization;
+ this.v2Serialization = v2Serialization;
+ this.managerCache = managerCache;
+ this.keyspace = keyspace;
+ }
+
+
+ @Override
+ public void migrate( final ProgressObserver observer ) throws Throwable {
+
+
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 ).doOnNext(
+ new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
+
+
+ @Override
+ public void call(
+ final AllEntitiesInSystemObservable.ApplicationEntityGroup applicationEntityGroup ) {
+
+
+ final UUID now = UUIDGenerator.newTimeUUID();
+
+ final Id appScopeId = applicationEntityGroup.applicationScope.getApplication();
+
+
+ final MutationBatch totalBatch = keyspace.prepareMutationBatch();
+
+ for ( Id entityId : applicationEntityGroup.entityIds ) {
+
+ CollectionScope currentScope = CpNamingUtils.getCollectionScopeNameFromEntityType(
+ appScopeId, entityId.getType() );
+
+
+ Iterator<MvccEntity> allVersions =
+ v1Serialization.loadDescendingHistory( currentScope, entityId, now, 1000 );
+
+ while ( allVersions.hasNext() ) {
+ final MvccEntity version = allVersions.next();
+
+ final MutationBatch versionBatch = v2Serialization.write( currentScope, version );
+
+ totalBatch.mergeShallow( versionBatch );
+
+ if ( totalBatch.getRowCount() >= 50 ) {
+ try {
+ totalBatch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new DataMigrationException( "Unable to migrate batches ", e );
+ }
+ }
+ }
+ }
+
+ try {
+ totalBatch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new DataMigrationException( "Unable to migrate batches ", e );
+ }
+ }
+ } ).toBlocking().last();
+ }
+
+
+ @Override
+ public int getVersion() {
+ return Versions.VERSION_3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ed79642/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
index 1adfe73..8089dfd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigration.java
@@ -29,13 +29,10 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.map.MapManager;
-import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.map.MapScope;
-import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
@@ -47,9 +44,6 @@ import rx.functions.Action1;
*/
public class EntityTypeMappingMigration implements DataMigration {
-
- private static final Logger logger = LoggerFactory.getLogger( EntityTypeMappingMigration.class );
-
private final ManagerCache managerCache;
@@ -65,25 +59,27 @@ public class EntityTypeMappingMigration implements DataMigration {
final AtomicLong atomicLong = new AtomicLong();
- AllEntitiesInSystemObservable.getAllEntitiesInSystem(managerCache )
- .doOnNext( new Action1<AllEntitiesInSystemObservable.EntityData>() {
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem(managerCache, 1000 )
+ .doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
@Override
- public void call( final AllEntitiesInSystemObservable.EntityData entityData ) {
+ public void call( final AllEntitiesInSystemObservable.ApplicationEntityGroup applicationEntityGroup ) {
- final MapScope ms = CpNamingUtils.getEntityTypeMapScope( entityData.applicationScope.getApplication() );
+ final MapScope ms = CpNamingUtils.getEntityTypeMapScope( applicationEntityGroup.applicationScope.getApplication() );
final MapManager mapManager = managerCache.getMapManager( ms );
- final UUID entityUuid = entityData.entityId.getUuid();
- final String entityType = entityData.entityId.getType();
+ for(Id entityId: applicationEntityGroup.entityIds) {
+ final UUID entityUuid = entityId.getUuid();
+ final String entityType = entityId.getType();
- mapManager.putString( entityUuid.toString(), entityType );
+ mapManager.putString( entityUuid.toString(), entityType );
- if ( atomicLong.incrementAndGet() % 100 == 0 ) {
- updateStatus( atomicLong, observer );
+ if ( atomicLong.incrementAndGet() % 100 == 0 ) {
+ updateStatus( atomicLong, observer );
+ }
}
}
} ).toBlocking().lastOrDefault( null );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ed79642/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java
index ac4cd58..3b92570 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigration.java
@@ -36,6 +36,7 @@ import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
+import org.apache.usergrid.persistence.model.entity.Id;
import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
@@ -64,8 +65,7 @@ public class GraphShardVersionMigration implements DataMigration {
@Inject
public GraphShardVersionMigration( @CurrentImpl final EdgeMetadataSerialization v2Serialization,
- final ManagerCache managerCache, final
- Keyspace keyspace ) {
+ final ManagerCache managerCache, final Keyspace keyspace ) {
this.v2Serialization = v2Serialization;
this.managerCache = managerCache;
this.keyspace = keyspace;
@@ -77,51 +77,71 @@ public class GraphShardVersionMigration implements DataMigration {
final AtomicLong counter = new AtomicLong();
- AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache).flatMap(
- new Func1<AllEntitiesInSystemObservable.EntityData, Observable<List<Edge>>>() {
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 ).flatMap(
+ new Func1<AllEntitiesInSystemObservable.ApplicationEntityGroup, Observable<List<Edge>>>() {
@Override
- public Observable<List<Edge>> call( final AllEntitiesInSystemObservable.EntityData entityData ) {
- logger.info( "Migrating edges from node {} in scope {}", entityData.entityId,
- entityData.applicationScope );
-
- final GraphManager gm = managerCache.getGraphManager( entityData.applicationScope );
-
- //get each edge from this node as a source
- return EdgesFromSourceObservable.edgesFromSource( gm, entityData.entityId )
-
- //for each edge, re-index it in v2 every 1000 edges or less
- .buffer( 1000 ).doOnNext( new Action1<List<Edge>>() {
- @Override
- public void call( final List<Edge> edges ) {
-
- final MutationBatch batch = keyspace.prepareMutationBatch();
-
- for ( final Edge edge : edges ) {
- logger.info( "Migrating meta for edge {}", edge );
- final MutationBatch edgeBatch =
- v2Serialization.writeEdge( entityData.applicationScope, edge );
- batch.mergeShallow( edgeBatch );
- }
-
- try {
- batch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to perform migration", e );
- }
-
- //update the observer so the admin can see it
- final long newCount = counter.addAndGet( edges.size() );
-
- observer.update( getVersion(), String.format("Currently running. Rewritten %d edge types", newCount) );
-
-
- }
- } );
+ public Observable<List<Edge>> call(
+ final AllEntitiesInSystemObservable.ApplicationEntityGroup applicationEntityGroup ) {
+
+ //emit a stream of all ids from this group
+ return Observable.from( applicationEntityGroup.entityIds )
+ .flatMap( new Func1<Id, Observable<List<Edge>>>() {
+
+
+ //for each id in the group, get it's edges
+ @Override
+ public Observable<List<Edge>> call( final Id id ) {
+ logger.info( "Migrating edges from node {} in scope {}", id,
+ applicationEntityGroup.applicationScope );
+
+ final GraphManager gm = managerCache
+ .getGraphManager( applicationEntityGroup.applicationScope );
+
+ //get each edge from this node as a source
+ return EdgesFromSourceObservable.edgesFromSource( gm, id )
+
+ //for each edge, re-index it in v2 every 1000 edges or less
+ .buffer( 1000 ).doOnNext( new Action1<List<Edge>>() {
+ @Override
+ public void call( final List<Edge> edges ) {
+
+ final MutationBatch batch =
+ keyspace.prepareMutationBatch();
+
+ for ( final Edge edge : edges ) {
+ logger.info( "Migrating meta for edge {}", edge );
+ final MutationBatch edgeBatch = v2Serialization
+ .writeEdge(
+ applicationEntityGroup
+ .applicationScope,
+ edge );
+ batch.mergeShallow( edgeBatch );
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException(
+ "Unable to perform migration", e );
+ }
+
+ //update the observer so the admin can see it
+ final long newCount =
+ counter.addAndGet( edges.size() );
+
+ observer.update( getVersion(), String.format(
+ "Currently running. Rewritten %d edge types",
+ newCount ) );
+ }
+ } );
+ }
+ } );
}
} ).toBlocking().lastOrDefault( null );
+ ;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ed79642/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservable.java
index 291bbe9..771b81f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservable.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.rx;
+import java.util.List;
+
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
@@ -40,14 +42,17 @@ public class AllEntitiesInSystemObservable {
/**
* Return an observable that emits all entities in the system.
+ * @param managerCache the managerCache to use
+ * @param bufferSize The amount of entityIds to buffer into each ApplicationEntityGroup. Note that if we exceed the buffer size
+ * you may be more than 1 ApplicationEntityGroup with the same application and different ids
*/
- public static Observable<EntityData> getAllEntitiesInSystem( final ManagerCache managerCache) {
+ public static Observable<ApplicationEntityGroup> getAllEntitiesInSystem( final ManagerCache managerCache, final int bufferSize) {
//traverse all nodes in the graph, load all source edges from them, then re-save the meta data
return ApplicationObservable.getAllApplicationIds( managerCache )
- .flatMap( new Func1<Id, Observable<EntityData>>() {
+ .flatMap( new Func1<Id, Observable<ApplicationEntityGroup>>() {
@Override
- public Observable<EntityData> call( final Id applicationId ) {
+ public Observable<ApplicationEntityGroup> call( final Id applicationId ) {
//set up our application scope and graph manager
final ApplicationScope applicationScope = new ApplicationScopeImpl(
@@ -68,11 +73,11 @@ public class AllEntitiesInSystemObservable {
//merge both the specified application node and the entity node
// so they all get used
- return Observable.merge( applicationNode, entityNodes )
- .map( new Func1<Id, EntityData>() {
+ return Observable.merge( applicationNode, entityNodes ).buffer(bufferSize)
+ .map( new Func1<List<Id>, ApplicationEntityGroup>() {
@Override
- public EntityData call( final Id id ) {
- return new EntityData( applicationScope, id );
+ public ApplicationEntityGroup call( final List<Id> id ) {
+ return new ApplicationEntityGroup( applicationScope, id );
}
} );
}
@@ -83,14 +88,14 @@ public class AllEntitiesInSystemObservable {
/**
* Get the entity data. Immutable bean for fast access
*/
- public static final class EntityData {
+ public static final class ApplicationEntityGroup {
public final ApplicationScope applicationScope;
- public final Id entityId;
+ public final List<Id> entityIds;
- public EntityData( final ApplicationScope applicationScope, final Id entityId ) {
+ public ApplicationEntityGroup( final ApplicationScope applicationScope, final List<Id> entityIds ) {
this.applicationScope = applicationScope;
- this.entityId = entityId;
+ this.entityIds = entityIds;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ed79642/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index fa9f9df..9d0c9e6 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -52,6 +52,7 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import com.fasterxml.uuid.UUIDComparator;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -220,9 +221,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
EntityManager em = app.getEntityManager();
- CollectionScope cs = new CollectionScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
- new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
- CpNamingUtils.getCollectionScopeNameFromEntityType( eref.getType() ) );
+ CollectionScope cs = getCollectionScopeNameFromEntityType( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ), eref.getType() );
EntityCollectionManagerFactory ecmf = CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ed79642/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
index dafdb00..1f0665a 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityTypeMappingMigrationIT.java
@@ -35,7 +35,6 @@ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.persistence.map.impl.MapSerializationImpl;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -46,7 +45,6 @@ import rx.functions.Action1;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -75,7 +73,7 @@ public class EntityTypeMappingMigrationIT extends AbstractCoreIT {
@Test
public void testIdMapping() throws Throwable {
- assertEquals("version 1 expected", 1, entityTypeMappingMigration.getVersion());
+ assertEquals( "version 1 expected", 1, entityTypeMappingMigration.getVersion() );
final EntityManager newAppEm = app.getEntityManager();
@@ -87,7 +85,6 @@ public class EntityTypeMappingMigrationIT extends AbstractCoreIT {
final Set<Id> type2Identities = EntityWriteHelper.createTypes( newAppEm, type2, size );
-
final Set<Id> allEntities = new HashSet<>();
allEntities.addAll( type1Identities );
allEntities.addAll( type2Identities );
@@ -106,38 +103,45 @@ public class EntityTypeMappingMigrationIT extends AbstractCoreIT {
entityTypeMappingMigration.migrate( progressObserver );
-
-
-
- AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache )
- .doOnNext( new Action1<AllEntitiesInSystemObservable.EntityData>() {
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 )
+ .doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
@Override
- public void call( final AllEntitiesInSystemObservable.EntityData entity ) {
+ public void call(
+ final AllEntitiesInSystemObservable.ApplicationEntityGroup entity ) {
//ensure that each one has a type
- try {
-
- final EntityManager em = emf.getEntityManager( entity.applicationScope.getApplication().getUuid() );
- final Entity returned = em.get( entity.entityId.getUuid() );
-
- //we seem to occasionally get phantom edges. If this is the case we'll store the type _> uuid mapping, but we won't have anything to load
- if(returned != null) {
- assertEquals( entity.entityId.getUuid(), returned.getUuid() );
- assertEquals( entity.entityId.getType(), returned.getType() );
- }
- else {
- final String type = managerCache.getMapManager( CpNamingUtils.getEntityTypeMapScope(
- entity.applicationScope.getApplication() ) )
- .getString( entity.entityId.getUuid().toString() );
-
- assertEquals(entity.entityId.getType(), type);
- }
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Unable to get entity " + entity.entityId
- + " by UUID, migration failed", e );
- }
- allEntities.remove( entity.entityId );
+ final EntityManager em = emf.getEntityManager(
+ entity.applicationScope.getApplication().getUuid() );
+
+ for ( final Id id : entity.entityIds ) {
+ try {
+ final Entity returned = em.get( id.getUuid() );
+
+ //we seem to occasionally get phantom edges. If this is the
+ // case we'll store the type _> uuid mapping, but we won't have
+ // anything to load
+
+ if ( returned != null ) {
+ assertEquals( id.getUuid(), returned.getUuid() );
+ assertEquals( id.getType(), returned.getType() );
+ }
+ else {
+ final String type = managerCache.getMapManager( CpNamingUtils
+ .getEntityTypeMapScope(
+ entity.applicationScope.getApplication() ) )
+ .getString( id.getUuid()
+ .toString() );
+
+ assertEquals( id.getType(), type );
+ }
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to get entity " + id
+ + " by UUID, migration failed", e );
+ }
+
+ allEntities.remove( id );
+ }
}
} ).toBlocking().lastOrDefault( null );
@@ -145,9 +149,5 @@ public class EntityTypeMappingMigrationIT extends AbstractCoreIT {
assertEquals( "Every element should have been encountered", 0, allEntities.size() );
assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
-
-
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ed79642/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
index aab47a0..88c02cd 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
@@ -32,12 +32,9 @@ import org.apache.usergrid.corepersistence.EntityWriteHelper;
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManagerImpl;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerializationImpl;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -114,40 +111,41 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
//read everything in previous version format and put it into our types.
- AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache )
- .doOnNext( new Action1<AllEntitiesInSystemObservable.EntityData>() {
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000)
+ .doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
@Override
- public void call( final AllEntitiesInSystemObservable.EntityData entity ) {
+ public void call( final AllEntitiesInSystemObservable.ApplicationEntityGroup entity ) {
final GraphManager gm =
managerCache.getGraphManager( entity.applicationScope );
- /**
- * Get our edge types from the source
- */
- gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( entity.entityId, null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- sourceTypes.put( entity.entityId, s );
- }
- } ).toBlocking().lastOrDefault( null );
-
-
- /**
- * Get the edge types to the target
- */
- gm.getEdgeTypesToTarget(
- new SimpleSearchEdgeType( entity.entityId, null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- targetTypes.put( entity.entityId, s );
- }
- } ).toBlocking().lastOrDefault( null );
-
- allEntities.remove( entity.entityId );
+ for(final Id id: entity.entityIds) {
+ /**
+ * Get our edge types from the source
+ */
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( id, null, null ) )
+ .doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String s ) {
+ sourceTypes.put( id, s );
+ }
+ } ).toBlocking().lastOrDefault( null );
+
+
+ /**
+ * Get the edge types to the target
+ */
+ gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( id,
+ null, null ) )
+ .doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String s ) {
+ targetTypes.put( id, s );
+ }
+ } ).toBlocking().lastOrDefault( null );
+
+ allEntities.remove( id );
+ }
}
} ).toBlocking().lastOrDefault( null );
@@ -169,42 +167,66 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
//now visit all nodes in the system and remove their types from the multi maps, it should be empty at the end
- AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache )
- .doOnNext( new Action1<AllEntitiesInSystemObservable.EntityData>() {
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 )
+ .doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
@Override
- public void call( final AllEntitiesInSystemObservable.EntityData entity ) {
+ public void call(
+ final AllEntitiesInSystemObservable.ApplicationEntityGroup entity ) {
final GraphManager gm =
managerCache.getGraphManager( entity.applicationScope );
- /**
- * Get our edge types from the source
- */
- gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( entity.entityId, null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- sourceTypes.remove( entity.entityId, s );
- }
- } ).toBlocking().lastOrDefault( null );
-
-
- /**
- * Get the edge types to the target
- */
- gm.getEdgeTypesToTarget(
- new SimpleSearchEdgeType( entity.entityId, null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- targetTypes.remove( entity.entityId, s );
- }
- } ).toBlocking().lastOrDefault( null );
+ for ( final Id id : entity.entityIds ) {
+ /**
+ * Get our edge types from the source
+ */
+ gm.getEdgeTypesFromSource(
+ new SimpleSearchEdgeType( id, null, null ) )
+ .doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String s ) {
+ sourceTypes.remove( id, s );
+ }
+ } ).toBlocking().lastOrDefault( null );
+
+
+ /**
+ * Get the edge types to the target
+ */
+ gm.getEdgeTypesToTarget(
+ new SimpleSearchEdgeType( id, null, null ) )
+ .doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String s ) {
+ targetTypes.remove( id, s );
+ }
+ } ).toBlocking().lastOrDefault( null );
+ }
+ }
}
- } ).toBlocking().lastOrDefault( null );
- assertEquals( "All source types migrated", 0, sourceTypes.size() );
- assertEquals( "All target types migrated", 0, targetTypes.size() );
+
+ ).
+
+
+ toBlocking()
+
+
+ .
+
+
+ lastOrDefault( null );
+
+
+ assertEquals( "All source types migrated",0,sourceTypes.size( )
+
+
+ );
+
+
+ assertEquals( "All target types migrated",0,targetTypes.size( )
+
+
+ );
+ }
}
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ed79642/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java
index 423dc1f..4d1c6c9 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/AllEntitiesInSystemObservableIT.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.corepersistence.rx;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
@@ -33,13 +32,11 @@ import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.corepersistence.EntityWriteHelper;
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import rx.functions.Action1;
@@ -95,26 +92,28 @@ public class AllEntitiesInSystemObservableIT extends AbstractCoreIT {
final GraphManager gm = managerCache.getGraphManager( scope );
- AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache ).doOnNext( new Action1<AllEntitiesInSystemObservable.EntityData>() {
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 ).doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
@Override
- public void call( final AllEntitiesInSystemObservable.EntityData entity ) {
+ public void call( final AllEntitiesInSystemObservable.ApplicationEntityGroup entity ) {
assertNotNull(entity);
assertNotNull(entity.applicationScope);
- assertNotNull(entity.entityId);
+ assertNotNull(entity.entityIds);
//not from our test, don't check it
if(!applicationId.equals( entity.applicationScope.getApplication() )){
return;
}
+ for(Id id: entity.entityIds) {
- //we should only emit each node once
- if ( entity.entityId.getType().equals( type1 ) ) {
- assertTrue( "Element should be present on removal", type1Identities.remove( entity.entityId ) );
- }
- else if ( entity.entityId.getType().equals( type2 ) ) {
- assertTrue( "Element should be present on removal", type2Identities.remove( entity.entityId ) );
+ //we should only emit each node once
+ if ( id.getType().equals( type1 ) ) {
+ assertTrue( "Element should be present on removal", type1Identities.remove( id ) );
+ }
+ else if ( id.getType().equals( type2 ) ) {
+ assertTrue( "Element should be present on removal", type2Identities.remove( id ) );
+ }
}
}
} ).toBlocking().lastOrDefault( null );
[15/19] incubator-usergrid git commit: Refactored tests.
Posted by sf...@apache.org.
Refactored tests.
Made configuration options easier to override in tests
Added checks for entity too large on write.
Changed default max load size to 100.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fb90ae70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fb90ae70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fb90ae70
Branch: refs/heads/two-dot-o
Commit: fb90ae705c29ee2cf61063d57f1e4e97d05ee5aa
Parents: e32d521
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 20 17:49:13 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 20 17:49:13 2014 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/GuiceModule.java | 2 +
.../org/apache/usergrid/CoreITSetupImpl.java | 13 ++
.../usergrid/persistence/LargeEntityIT.java | 30 +++-
.../exception/EntityTooLargeException.java | 67 ++++++++
.../serialization/SerializationFig.java | 14 +-
.../MvccEntitySerializationStrategyImpl.java | 41 ++---
.../MvccEntitySerializationStrategyV2Impl.java | 82 ++++++----
.../collection/EntityCollectionManagerIT.java | 16 +-
...MvccEntitySerializationStrategyImplTest.java | 52 +-----
...cEntitySerializationStrategyProxyV1Test.java | 17 --
...cEntitySerializationStrategyProxyV2Test.java | 19 +--
...ccEntitySerializationStrategyV1ImplTest.java | 18 --
...ccEntitySerializationStrategyV2ImplTest.java | 19 +--
.../MvccEntitySerializationStrategyV2Test.java | 164 +++++++++++++++++++
.../impl/MvccLESSTransientTest.java | 53 +-----
.../collection/util/EntityHelper.java | 35 ++--
.../persistence/core/astyanax/CassandraFig.java | 12 ++
.../core/guicyfig/SetConfigTestBypass.java | 108 ++++++++++++
.../persistence/queue/QueueManagerTest.java | 26 +--
19 files changed, 522 insertions(+), 266 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index 2d80672..018a9b7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
import com.google.inject.multibindings.Multibinder;
+import org.apache.usergrid.corepersistence.migration.EntityDataMigration;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
@@ -56,6 +57,7 @@ public class GuiceModule extends AbstractModule {
Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
+ dataMigrationMultibinder.addBinding().to( EntityDataMigration.class );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java b/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
index b3cc046..a5f59ed 100644
--- a/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
@@ -24,10 +24,13 @@ import org.junit.runners.model.Statement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.cassandra.CassandraResource;
+import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.mq.QueueManagerFactory;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.utils.JsonUtils;
@@ -90,7 +93,17 @@ public class CoreITSetupImpl implements CoreITSetup {
emf = cassandraResource.getBean( EntityManagerFactory.class );
qmf = cassandraResource.getBean( QueueManagerFactory.class );
indexBucketLocator = cassandraResource.getBean( IndexBucketLocator.class );
+
+ //run the migration
+ try {
+ CpSetup.getInjector().getInstance( DataMigrationManager.class ).migrate();
+ }
+ catch ( MigrationException e ) {
+ throw new RuntimeException( "Unable to run migration", e );
+ }
+
enabled = true;
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
index eef2820..c95f7d6 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
@@ -25,7 +25,7 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Map;
-import org.aspectj.lang.annotation.Before;
+import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
@@ -35,6 +35,9 @@ import org.apache.usergrid.AbstractCoreIT;
import org.apache.usergrid.Application;
import org.apache.usergrid.CoreApplication;
import org.apache.usergrid.cassandra.Concurrent;
+import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
import org.apache.usergrid.utils.JsonUtils;
import static org.junit.Assert.assertEquals;
@@ -48,6 +51,28 @@ public class LargeEntityIT extends AbstractCoreIT {
public Application app = new CoreApplication( setup );
+ /**
+ * Set our max size before and after so we can override it in our runtime tests
+ */
+ private int setMaxEntitySize;
+
+ private SerializationFig serializationFig;
+
+
+ @org.junit.Before
+ public void setUp() {
+
+ serializationFig = CpSetup.getInjector().getInstance( SerializationFig.class );
+
+ setMaxEntitySize = serializationFig.getMaxEntitySize();
+ }
+
+
+ @After
+ public void tearDown() {
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", setMaxEntitySize + "" );
+ }
+
/**
* Tests creating a large entity, then loading it, modifying it, saving it, then loading it again
@@ -57,6 +82,9 @@ public class LargeEntityIT extends AbstractCoreIT {
LOG.debug( "testLargeEntityCrud" );
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", 641814 + "" );
+
+
final URL resource = this.getClass().getClassLoader().getResource( TEST_DATA_FILE );
final byte[] fileData = Files.readAllBytes( Paths.get( resource.toURI() ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/EntityTooLargeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/EntityTooLargeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/EntityTooLargeException.java
new file mode 100644
index 0000000..11224db
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/EntityTooLargeException.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.collection.exception;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+
+public class EntityTooLargeException extends RuntimeException {
+
+ private final Entity entity;
+ private final int maxSize;
+ private final int attemptedSize;
+
+
+ public EntityTooLargeException( Entity entity, int maxSize,final int attemptedSize, final String message ) {
+ super( message );
+ this.entity = entity;
+ this.maxSize = maxSize;
+
+ this.attemptedSize = attemptedSize;
+ }
+
+
+ /**
+ * Get the max size allowed
+ * @return
+ */
+ public int getMaxSize() {
+ return maxSize;
+ }
+
+
+ /**
+ * Get the size of the entity we attempted to save
+ * @return
+ */
+ public int getAttemptedSize() {
+ return attemptedSize;
+ }
+
+
+ /**
+ * Entity involved in operation.
+ * @return Entity or null if entity not instantiated yet in operation.
+ */
+ public Entity getEntity() {
+ return entity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 294c7ab..0687312 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -58,17 +58,21 @@ public interface SerializationFig extends GuicyFig {
int getTaskPoolQueueSize();
/**
- * The maximum amount of entites we can load at any one time
+ * The maximum amount of entities we can load at any one time
*/
@Key( "collection.max.load.size" )
- @Default( "1000" )
+ @Default( "100" )
int getMaxLoadSize();
/**
* The maximum number of bytes a serialized entity can be. Any thing beyond this is rejected
+ * This default is based on the following equation
+ *
+ * (15mb thrift buffer * .9) / 100 (default max load size)
*/
- @Key( "collection.max.entry.size" )
- @Default( "512" )
- int getMaxEntrySize();
+ @Key( "collection.max.entity.size" )
+ @Default( "141557" )
+ int getMaxEntitySize();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index 4ac5d24..f3f4c13 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -77,7 +77,6 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
protected final Keyspace keyspace;
protected final SerializationFig serializationFig;
protected final EntityRepair repair;
- private final AbstractSerializer<EntityWrapper> entityJsonSerializer;
private final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> columnFamily;
@@ -86,7 +85,6 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
this.keyspace = keyspace;
this.serializationFig = serializationFig;
this.repair = new EntityRepairImpl( this, serializationFig );
- this.entityJsonSerializer = getEntitySerializer();
this.columnFamily = getColumnFamily();
}
@@ -102,17 +100,17 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
return doWrite( collectionScope, entityId, new RowOp() {
@Override
public void doOp( final ColumnListMutation<UUID> colMutation ) {
- try {
- colMutation.putColumn( colName, entityJsonSerializer
+// try {
+ colMutation.putColumn( colName, getEntitySerializer()
.toByteBuffer( new EntityWrapper( entity.getStatus(), entity.getEntity() ) ) );
- }
- catch ( Exception e ) {
- // throw better exception if we can
- if ( entity != null || entity.getEntity().get() != null ) {
- throw new CollectionRuntimeException( entity, collectionScope, e );
- }
- throw e;
- }
+// }
+// catch ( Exception e ) {
+// // throw better exception if we can
+// if ( entity != null || entity.getEntity().get() != null ) {
+// throw new CollectionRuntimeException( entity, collectionScope, e );
+// }
+// throw e;
+// }
}
} );
}
@@ -129,9 +127,9 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
Preconditions.checkNotNull( maxVersion, "version is required" );
- //didn't put the max in the error message, I don't want to take the string construction hit every time
- Preconditions.checkArgument( entityIds.size() <= serializationFig.getMaxLoadSize(),
- "requested size cannot be over configured maximum" );
+ if( entityIds.size() > serializationFig.getMaxLoadSize()){
+ throw new IllegalArgumentException( "requested load size cannot be over configured maximum of " + serializationFig.getMaxLoadSize() );
+ }
final Id applicationId = collectionScope.getApplication();
@@ -185,7 +183,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
final Column<UUID> column = columns.getColumnByIndex( 0 );
final MvccEntity parsedEntity =
- new MvccColumnParser( entityId, entityJsonSerializer ).parseColumn( column );
+ new MvccColumnParser( entityId, getEntitySerializer() ).parseColumn( column );
//we *might* need to repair, it's not clear so check before loading into result sets
final MvccEntity maybeRepaired = repair.maybeRepair( collectionScope, parsedEntity );
@@ -223,7 +221,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
keyspace.prepareQuery( columnFamily ).getKey( rowKey )
.withColumnRange( version, null, false, fetchSize );
- return new ColumnNameIterator( query, new MvccColumnParser( entityId, entityJsonSerializer ), false );
+ return new ColumnNameIterator( query, new MvccColumnParser( entityId, getEntitySerializer() ), false );
}
@@ -253,7 +251,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
keyspace.prepareQuery( columnFamily ).getKey( rowKey )
.withColumnRange( null, version, true, fetchSize );
- return new ColumnNameIterator( query, new MvccColumnParser( entityId, entityJsonSerializer ), false );
+ return new ColumnNameIterator( query, new MvccColumnParser( entityId, getEntitySerializer() ), false );
}
@@ -268,7 +266,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
return doWrite( collectionScope, entityId, new RowOp() {
@Override
public void doOp( final ColumnListMutation<UUID> colMutation ) {
- colMutation.putColumn( version, entityJsonSerializer
+ colMutation.putColumn( version, getEntitySerializer()
.toByteBuffer( new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() ) ) );
}
} );
@@ -410,10 +408,5 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
* @return
*/
protected abstract MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> getColumnFamily();
- /**
- *
- private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> CF_ENTITY_DATA =
- new MultiTennantColumnFamily<>( "Entity_Version_Data", ROW_KEY_SER, UUIDSerializer.get() );
- */
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
index 723bb38..923c399 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -25,6 +25,7 @@ import java.util.UUID;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
+import org.apache.usergrid.persistence.collection.exception.EntityTooLargeException;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.astyanax.FieldBuffer;
import org.apache.usergrid.persistence.core.astyanax.FieldBufferBuilder;
@@ -36,6 +37,7 @@ import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.google.common.base.Optional;
@@ -52,8 +54,6 @@ import com.netflix.astyanax.serializers.UUIDSerializer;
@Singleton
public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializationStrategyImpl {
- private static final EntitySerializer ENTITY_JSON_SER = new EntitySerializer();
-
private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
@@ -63,21 +63,24 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> CF_ENTITY_DATA =
- new MultiTennantColumnFamily<>( "Entity_Version_Data_V2", ROW_KEY_SER, UUIDSerializer.get() );
+ new MultiTennantColumnFamily<>( "Entity_Version_Data_V2", ROW_KEY_SER, UUIDSerializer.get() );
private static final FieldBufferSerializer FIELD_BUFFER_SERIALIZER = FieldBufferSerializer.get();
+ private final EntitySerializer entitySerializer;
+
@Inject
public MvccEntitySerializationStrategyV2Impl( final Keyspace keyspace, final SerializationFig serializationFig ) {
super( keyspace, serializationFig );
+ entitySerializer = new EntitySerializer( serializationFig );
}
@Override
protected AbstractSerializer<EntityWrapper> getEntitySerializer() {
- return ENTITY_JSON_SER;
+ return entitySerializer;
}
@@ -87,35 +90,33 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
}
- public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
+ /**
+ * We should only ever create this once, since this impl is a singleton
+ */
+ public final class EntitySerializer extends AbstractSerializer<EntityWrapper> {
-// private static final ByteBufferSerializer BUFFER_SERIALIZER = ByteBufferSerializer.get();
-//
-// private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+ private final SmileFactory SMILE_FACTORY = new SmileFactory();
+ private final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
- public static final SmileFactory f = new SmileFactory();
- public static ObjectMapper mapper;
+ private SerializationFig serializationFig;
- private static byte STATE_COMPLETE = 0 ;
- private static byte STATE_DELETED = 1 ;
- private static byte STATE_PARTIAL = 2 ;
- private static byte VERSION = 1;
+ private byte STATE_COMPLETE = 0;
+ private byte STATE_DELETED = 1;
+ private byte STATE_PARTIAL = 2;
+ private byte VERSION = 1;
- public EntitySerializer() {
- try {
- mapper = new ObjectMapper( f );
- // mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
- // causes slowness
- mapper.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Error setting up mapper", e );
- }
+
+ public EntitySerializer( final SerializationFig serializationFig ) {
+ this.serializationFig = serializationFig;
+
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
+ // causes slowness
+ MAPPER.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
}
@@ -126,7 +127,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
}
//we always have a max of 3 fields
- FieldBufferBuilder builder = new FieldBufferBuilder( 3 );
+ FieldBufferBuilder builder = new FieldBufferBuilder( 3 );
builder.addByte( VERSION );
@@ -149,14 +150,28 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
builder.addByte( STATE_PARTIAL );
}
+
+ final Entity entity = wrapper.entity.get();
+ final byte[] entityBytes;
+
try {
- final byte[] entityBytes = mapper.writeValueAsBytes( wrapper.entity.get() );
- builder.addBytes( entityBytes );
+ entityBytes = MAPPER.writeValueAsBytes( entity );
}
- catch ( Exception e ) {
+ catch ( JsonProcessingException e ) {
throw new RuntimeException( "Unable to serialize entity", e );
}
+
+ final int maxEntrySize = serializationFig.getMaxEntitySize();
+
+ if ( entityBytes.length > maxEntrySize ) {
+ throw new EntityTooLargeException( entity, maxEntrySize, entityBytes.length,
+ "Your entity cannot exceed " + maxEntrySize + " bytes. The entity you tried to save was "
+ + entityBytes.length + " bytes" );
+ }
+
+ builder.addBytes( entityBytes );
+
return FIELD_BUFFER_SERIALIZER.toByteBuffer( builder.build() );
}
@@ -181,13 +196,12 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
throw new DataCorruptionException( "Unable to de-serialze entity", e );
}
- FieldBufferParser parser = new FieldBufferParser( fieldBuffer );
-
+ FieldBufferParser parser = new FieldBufferParser( fieldBuffer );
byte version = parser.readByte();
- if ( VERSION != version ) {
+ if ( VERSION != version ) {
throw new UnsupportedOperationException( "A version of type " + version + " is unsupported" );
}
@@ -195,7 +209,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
// it's been deleted, remove it
- if ( STATE_DELETED == state ) {
+ if ( STATE_DELETED == state ) {
return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
}
@@ -204,7 +218,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
byte[] array = parser.readBytes();
try {
- storedEntity = mapper.readValue( array, Entity.class );
+ storedEntity = MAPPER.readValue( array, Entity.class );
}
catch ( Exception e ) {
throw new DataCorruptionException( "Unable to read entity data", e );
@@ -212,7 +226,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
final Optional<Entity> entity = Optional.of( storedEntity );
- if ( STATE_COMPLETE == state ) {
+ if ( STATE_COMPLETE == state ) {
return new EntityWrapper( MvccEntity.Status.COMPLETE, entity );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 21e3260..5eba979 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -28,12 +28,13 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
-import org.apache.usergrid.persistence.collection.util.EntityHelper;
-import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.core.util.Health;
@@ -713,8 +714,13 @@ public class EntityCollectionManagerIT {
public void largeEntityWriteRead() {
final int setSize = 65535 * 2;
+ final int currentMaxSize = serializationFig.getMaxEntitySize();
- final Entity entity = EntityHelper.generateEntity( setSize );
+ //override our default
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", 65535 * 10 + "" );
+
+
+ final Entity entity = EntityHelper.generateEntity( setSize );
//now we have one massive, entity, save it and retrieve it.
CollectionScope context =
@@ -732,5 +738,9 @@ public class EntityCollectionManagerIT {
EntityHelper.verifyDeepEquals( entity, loaded );
+
+
+ //override our default
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", currentMaxSize + "" );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index 0cdc320..6ce88cf 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -75,7 +75,7 @@ import static org.mockito.Mockito.mock;
public abstract class MvccEntitySerializationStrategyImplTest {
- private MvccEntitySerializationStrategy serializationStrategy;
+ protected MvccEntitySerializationStrategy serializationStrategy;
@Inject
@@ -582,56 +582,6 @@ public abstract class MvccEntitySerializationStrategyImplTest {
- /**
- * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
- */
- @Test
- public void largeEntityWriteRead() throws ConnectionException {
- final int setSize = 65535 * 2;
-
- final Entity entity = EntityHelper.generateEntity( setSize );
-
- //now we have one massive, entity, save it and retrieve it.
- CollectionScope context =
- new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "parent" ), "tests" );
-
-
- final Id id = entity.getId();
- ValidationUtils.verifyIdentity( id );
- final UUID version = UUIDGenerator.newTimeUUID();
- final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
-
- final MvccEntity mvccEntity = new MvccEntityImpl( id, version, status, entity );
-
-
- getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
-
- //now load it
- final Iterator<MvccEntity> loaded =
- getMvccEntitySerializationStrategy().loadDescendingHistory( context, id, version, 100 );
-
-
- assertLargeEntity( mvccEntity, loaded );
-
- MvccEntity returned = serializationStrategy.load( context, Collections.singleton( id ) , version ).getEntity( id );
-
- assertLargeEntity( mvccEntity, returned );
- }
-
-
- /**
- * Assertion for the loaded values of a large entity. Note the returned may be nullable
- * @param expected
- * @param returned
- */
- protected abstract void assertLargeEntity(final MvccEntity expected, final Iterator<MvccEntity> returned);
-
- /**
- * Tests large entity that is directly returned
- * @param expected
- * @param returned
- */
- protected abstract void assertLargeEntity(final MvccEntity expected, final MvccEntity returned);
@Test(expected = NullPointerException.class)
public void writeParamsContext() throws ConnectionException {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
index df5f7d1..a936d2a 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
@@ -76,23 +76,6 @@ public class MvccEntitySerializationStrategyProxyV1Test extends MvccEntitySerial
- @Override
- protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
- //known bug in v1, it should return null. Fixed in v2
- assertTrue( "V1 should be broken, but still have a next", returned.hasNext() );
-
- final MvccEntity entity = returned.next();
-
-
- assertLargeEntity( expected, entity );
- }
-
-
- protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
- assertEquals( "Marked as deleted since we can't parse", MvccEntity.Status.DELETED, returned.getStatus() );
-
- assertFalse( "V1 returns empty b/c parse fails", returned.getEntity().isPresent() );
- }
@After
public void reSetMigrationVersion() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
index dc31ff7..1709cc1 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
@@ -42,7 +42,7 @@ import static org.junit.Assert.assertTrue;
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerializationStrategyImplTest {
+public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerializationStrategyV2Test {
@Inject
@ProxyImpl
@@ -74,23 +74,6 @@ public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerial
}
- @Override
- protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
- assertTrue( returned.hasNext() );
-
- final MvccEntity loadedEntity = returned.next();
-
- assertLargeEntity( expected, loadedEntity );
- }
-
-
- @Override
- protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
-
- org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
-
- EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
- }
@After
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
index 9b0ccfb..e59426d 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
@@ -47,24 +47,6 @@ public class MvccEntitySerializationStrategyV1ImplTest extends MvccEntitySeriali
private MvccEntitySerializationStrategy serializationStrategy;
- @Override
- protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
- //known bug in v1, it should return null. Fixed in v2
- assertTrue( "V1 should be broken, but still have a next", returned.hasNext() );
-
- final MvccEntity entity = returned.next();
-
-
- assertLargeEntity( expected, entity );
- }
-
-
- protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
- assertEquals( "Marked as deleted since we can't parse", MvccEntity.Status.DELETED, returned.getStatus() );
-
- assertFalse( "V1 returns empty b/c parse fails", returned.getEntity().isPresent() );
- }
-
@Override
protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
index 53952d0..c931904 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
@@ -39,7 +39,7 @@ import static org.junit.Assert.assertTrue;
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySerializationStrategyImplTest {
+public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySerializationStrategyV2Test {
@Inject
@CurrentImpl
@@ -52,21 +52,4 @@ public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySeriali
}
- @Override
- protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
- assertTrue( returned.hasNext() );
-
- final MvccEntity loadedEntity = returned.next();
-
- assertLargeEntity( expected, loadedEntity );
- }
-
-
- @Override
- protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
-
- org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
-
- EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java
new file mode 100644
index 0000000..a496519
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import java.lang.annotation.Annotation;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.safehaus.guicyfig.Bypass;
+import org.safehaus.guicyfig.Env;
+import org.safehaus.guicyfig.Option;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.exception.EntityTooLargeException;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
+import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import static org.junit.Assert.assertTrue;
+
+
+public abstract class MvccEntitySerializationStrategyV2Test extends MvccEntitySerializationStrategyImplTest {
+
+
+ @Inject
+ protected SerializationFig serializationFig;
+
+ private int setMaxEntitySize;
+
+
+ @Before
+ public void setUp(){
+
+
+ setMaxEntitySize = serializationFig.getMaxEntitySize();
+ }
+
+ @After
+ public void tearDown(){
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", setMaxEntitySize + "" );
+ }
+
+ /**
+ * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
+ */
+ @Test
+ public void largeEntityWriteRead() throws ConnectionException {
+ final int setSize = 65535 * 2;
+
+
+ //this is the size it works out to be when serialized, we want to allow this size
+
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", 65535*10+"");
+ final Entity entity = EntityHelper.generateEntity( setSize );
+
+ //now we have one massive, entity, save it and retrieve it.
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "parent" ), "tests" );
+
+
+ final Id id = entity.getId();
+ ValidationUtils.verifyIdentity( id );
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
+
+ final MvccEntity mvccEntity = new MvccEntityImpl( id, version, status, entity );
+
+
+ getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
+
+ //now load it
+ final Iterator<MvccEntity> loaded =
+ getMvccEntitySerializationStrategy().loadDescendingHistory( context, id, version, 100 );
+
+
+ assertLargeEntity( mvccEntity, loaded );
+
+ MvccEntity returned =
+ serializationStrategy.load( context, Collections.singleton( id ), version ).getEntity( id );
+
+ assertLargeEntity( mvccEntity, returned );
+ }
+
+
+
+ /**
+ * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
+ */
+ @Test( expected = EntityTooLargeException.class )
+ public void entityLargerThanAllowedWrite() throws ConnectionException {
+ final int setSize = serializationFig.getMaxEntitySize() + 1;
+
+ final Entity entity = EntityHelper.generateEntity( setSize );
+
+ //now we have one massive, entity, save it and retrieve it.
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "parent" ), "tests" );
+
+
+ final Id id = entity.getId();
+ ValidationUtils.verifyIdentity( id );
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
+
+ final MvccEntity mvccEntity = new MvccEntityImpl( id, version, status, entity );
+
+
+ getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
+ }
+
+
+ protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
+ assertTrue( returned.hasNext() );
+
+ final MvccEntity loadedEntity = returned.next();
+
+ assertLargeEntity( expected, loadedEntity );
+ }
+
+
+ protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+
+ org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
+
+ EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
index 4f96594..09b350c 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
@@ -24,6 +24,7 @@ import java.lang.annotation.Annotation;
import java.util.Collections;
import java.util.UUID;
+import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -41,6 +42,7 @@ import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerialization
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -80,58 +82,15 @@ public class MvccLESSTransientTest {
public void setTimeout() {
originalTimeout = serializationFig.getTimeout();
//set the bypass options
- serializationFig.setBypass( new TestByPass() );
+ SetConfigTestBypass.setValueByPass( serializationFig,"getTimeout", "1" );
}
-
- /**
- * Test bypass that sets all environments to use the timeout of 1 second
- */
- public class TestByPass implements Bypass {
-
-
- @Override
- public Option[] options() {
- return new Option[] { new TestOption() };
- }
-
-
- @Override
- public Env[] environments() {
- return new Env[] { Env.ALL, Env.UNIT };
- }
-
-
- @Override
- public Class<? extends Annotation> annotationType() {
- return Bypass.class;
- }
+ @After
+ public void resetTimeout(){
+ SetConfigTestBypass.setValueByPass( serializationFig,"getTimeout", originalTimeout + "" );
}
- /**
- * TestOption
- */
- public class TestOption implements Option {
-
-
- @Override
- public Class<? extends Annotation> annotationType() {
- return Bypass.class;
- }
-
-
- @Override
- public String method() {
- return "getTimeout";
- }
-
-
- @Override
- public String override() {
- return "1";
- }
- }
@Test
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
index 5c99121..4dafbaf 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
@@ -40,34 +40,33 @@ import static org.junit.Assert.assertNotNull;
public class EntityHelper {
/**
- * Generates an entity with all string fields to have at least the minSize
- * number of characters in the field names + field values
- * @param minSize
- * @return
+ * Generates an entity with all string fields to have at least the minSize number of characters in the field names +
+ * field values
*/
- public static Entity generateEntity(final int minSize){
+ public static Entity generateEntity( final int minSize ) {
int currentLength = 0;
- final Entity entity = new Entity( new SimpleId( "test" ) );
+ final Entity entity = new Entity( new SimpleId( "test" ) );
- //generate a really large string value
- StringBuilder builder = new StringBuilder();
+ //generate a really large string value
- for ( int i = 0; i < 100; i++ ) {
- builder.append( UUIDGenerator.newTimeUUID().toString() );
- }
- final String value = builder.toString();
+ //loop until our size is beyond the set size
+ for ( int i = 0; currentLength < minSize; i++ ) {
+ final String key = "newStringField" + i;
+ currentLength += key.length();
- //loop until our size is beyond the set size
- for ( int i = 0; currentLength < minSize; i++ ) {
- final String key = "newStringField" + i;
+ StringBuilder builder = new StringBuilder();
- entity.setField( new StringField( key, value ) );
+ for ( int j = 0; j < 1000 && currentLength < minSize; j++ ) {
+ builder.append( "a" );
+ currentLength ++;
+ }
- currentLength += key.length() + value.length();
- }
+
+ entity.setField( new StringField( key, builder.toString() ) );
+ }
return entity;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
index 4cf76f4..df94adc 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
@@ -39,6 +39,9 @@ public interface CassandraFig extends GuicyFig {
public static final String SHARD_VALUES = "cassandra.shardvalues";
+ public static final String THRIFT_TRANSPORT_SIZE = "cassandra.thrift.transport.frame";
+
+
@Key( "cassandra.hosts" )
String getHosts();
@@ -94,5 +97,14 @@ public interface CassandraFig extends GuicyFig {
@Key(SHARD_VALUES)
String getShardValues();
+ /**
+ * Get the thrift transport size. Should be set to what is on the cassandra servers. As we move to CQL, this will become obsolete
+ * @return
+ */
+ @Key( THRIFT_TRANSPORT_SIZE)
+ @Default( "15728640" )
+ int getThriftBufferSize();
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guicyfig/SetConfigTestBypass.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guicyfig/SetConfigTestBypass.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guicyfig/SetConfigTestBypass.java
new file mode 100644
index 0000000..cb28449
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/guicyfig/SetConfigTestBypass.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.guicyfig;
+
+
+import java.lang.annotation.Annotation;
+
+import org.safehaus.guicyfig.Bypass;
+import org.safehaus.guicyfig.Env;
+import org.safehaus.guicyfig.GuicyFig;
+import org.safehaus.guicyfig.Option;
+
+
+public class SetConfigTestBypass {
+
+
+ /**
+ * Set the value bypass on the guicyfig
+ * @param guicyFig
+ * @param methodName
+ * @param valueToSet
+ */
+ public static void setValueByPass(final GuicyFig guicyFig, final String methodName, final String valueToSet){
+ guicyFig.setBypass( new TestByPass( methodName, valueToSet ) );
+ }
+
+ /**
+ * Test bypass that sets all environments to use the timeout of 1 second
+ */
+ public static final class TestByPass implements Bypass {
+
+ private Option[] options;
+
+
+ public TestByPass( final String methodName, final String value ) {
+ options = new Option[] { new TestOption( methodName, value ) };
+ }
+
+
+ @Override
+ public Option[] options() {
+ return options;
+ }
+
+
+ @Override
+ public Env[] environments() {
+ return new Env[] { Env.ALL, Env.UNIT };
+ }
+
+
+ @Override
+ public Class<? extends Annotation> annotationType() {
+ return Bypass.class;
+ }
+ }
+
+
+ /**
+ * TestOption
+ */
+ public static final class TestOption implements Option {
+
+ private final String methodName;
+ private final String valueToReturn;
+
+
+ public TestOption( final String methodName, final String valueToReturn ) {
+ this.methodName = methodName;
+ this.valueToReturn = valueToReturn;
+ }
+
+
+ @Override
+ public Class<? extends Annotation> annotationType() {
+ return Bypass.class;
+ }
+
+
+ @Override
+ public String method() {
+ return methodName;
+ }
+
+
+ @Override
+ public String override() {
+ return valueToReturn;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fb90ae70/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
index 8eb5625..935fd16 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/QueueManagerTest.java
@@ -19,28 +19,30 @@
package org.apache.usergrid.persistence.queue;
-import org.apache.usergrid.persistence.queue.guice.TestQueueModule;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeFactoryImpl;
-import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-import org.apache.usergrid.persistence.core.test.UseModules;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.queue.guice.TestQueueModule;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeFactoryImpl;
+import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
import com.google.inject.Inject;
-import org.safehaus.guicyfig.Bypass;
-import org.safehaus.guicyfig.OptionState;
-import org.safehaus.guicyfig.Overrides;
-
-import java.beans.PropertyChangeListener;
-import java.io.IOException;
-import java.util.*;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
@RunWith( ITRunner.class )
@UseModules( { TestQueueModule.class } )
[05/19] incubator-usergrid git commit: Removed duplicate MigrationRule
Posted by sf...@apache.org.
Removed duplicate MigrationRule
Partially refactored entity serialization.
Finished creating tests for refactoring
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2bd1c950
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2bd1c950
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2bd1c950
Branch: refs/heads/two-dot-o
Commit: 2bd1c950f6fab093d458e6c3a55fb7fb8ff28e79
Parents: 39b1576
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 19 17:01:04 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 19 17:01:04 2014 -0700
----------------------------------------------------------------------
.../corepersistence/migration/Versions.java | 3 +
.../impl/EntityCollectionManagerImpl.java | 241 +++++++++----------
.../mvcc/MvccEntitySerializationStrategy.java | 3 +-
.../mvcc/MvccLogEntrySerializationStrategy.java | 3 +-
.../mvcc/stage/delete/MarkCommit.java | 3 +-
.../mvcc/stage/write/WriteCommit.java | 3 +-
.../UniqueValueSerializationStrategy.java | 3 +-
.../MvccEntitySerializationStrategyImpl.java | 172 ++-----------
...vccEntitySerializationStrategyProxyImpl.java | 162 +++++++++++++
.../MvccEntitySerializationStrategyV1Impl.java | 193 +++++++++++++++
.../MvccEntitySerializationStrategyV2Impl.java | 194 +++++++++++++++
.../MvccLogEntrySerializationStrategyImpl.java | 2 +-
.../serialization/impl/SerializationModule.java | 21 +-
.../UniqueValueSerializationStrategyImpl.java | 2 +-
.../EntityCollectionManagerFactoryTest.java | 2 +-
.../collection/EntityCollectionManagerIT.java | 74 ++++--
.../EntityCollectionManagerStressTest.java | 2 +-
.../EntityCollectionManagerSyncIT.java | 2 +-
.../collection/guice/MigrationManagerRule.java | 38 ---
.../collection/guice/TestCollectionModule.java | 14 ++
...niqueValueSerializationStrategyImplTest.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
...MvccEntitySerializationStrategyImplTest.java | 56 ++---
...cEntitySerializationStrategyProxyV1Test.java | 77 ++++++
...cEntitySerializationStrategyProxyV2Test.java | 75 ++++++
...ccEntitySerializationStrategyV1ImplTest.java | 46 ++++
...ccEntitySerializationStrategyV2ImplTest.java | 47 ++++
.../impl/MvccLESSTransientTest.java | 2 +-
...ccLogEntrySerializationStrategyImplTest.java | 2 +-
.../src/test/resources/log4j.properties | 21 +-
.../core/guice/MaxMigrationModule.java | 39 +++
.../core/guice/MaxMigrationVersion.java | 40 +++
.../core/guice/MigrationManagerRule.java | 19 +-
.../persistence/graph/GraphManagerIT.java | 2 +-
.../persistence/graph/GraphManagerLoadTest.java | 2 +-
.../graph/GraphManagerShardingIT.java | 2 +-
.../graph/GraphManagerStressTest.java | 2 +-
.../usergrid/persistence/graph/SimpleTest.java | 2 +-
.../graph/guice/TestGraphModule.java | 11 +
.../graph/impl/EdgeDeleteListenerTest.java | 2 +-
.../graph/impl/NodeDeleteListenerTest.java | 2 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 2 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 2 +-
.../EdgeMetadataSerializationTest.java | 2 +-
.../EdgeSerializationChopTest.java | 2 +-
.../serialization/EdgeSerializationTest.java | 2 +-
.../serialization/NodeSerializationTest.java | 2 +-
.../impl/shard/EdgeShardSerializationTest.java | 2 +-
.../NodeShardCounterSerializationTest.java | 2 +-
.../persistence/map/MapManagerTest.java | 2 +-
51 files changed, 1205 insertions(+), 405 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
index b4fe095..99067b7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/Versions.java
@@ -22,6 +22,7 @@
package org.apache.usergrid.corepersistence.migration;
+import org.apache.usergrid.persistence.collection.serialization.impl.MvccEntitySerializationStrategyProxyImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationProxyImpl;
@@ -39,4 +40,6 @@ public class Versions {
* Version 2. Edge meta changes
*/
public static final int VERSION_2 = EdgeMetadataSerializationProxyImpl.MIGRATION_VERSION;
+
+ public static final int VERSION_3 = MvccEntitySerializationStrategyProxyImpl.MIGRATION_VERSION;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 919e83b..d54c5f7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -23,23 +23,18 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
-import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
@@ -49,9 +44,16 @@ import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Preconditions;
@@ -59,11 +61,10 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.connectionpool.OperationResult;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.core.util.Health;
import rx.Observable;
import rx.Subscriber;
@@ -73,13 +74,12 @@ import rx.schedulers.Schedulers;
/**
- * Simple implementation. Should perform writes, delete and load.
- * <p/>
- * TODO: maybe refactor the stage operations into their own classes for clarity and organization?
+ * Simple implementation. Should perform writes, delete and load. <p/> TODO: maybe refactor the stage operations into
+ * their own classes for clarity and organization?
*/
public class EntityCollectionManagerImpl implements EntityCollectionManager {
- private static final Logger logger = LoggerFactory.getLogger(EntityCollectionManagerImpl.class);
+ private static final Logger logger = LoggerFactory.getLogger( EntityCollectionManagerImpl.class );
private final CollectionScope collectionScope;
@@ -106,24 +106,20 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Inject
- public EntityCollectionManagerImpl(
- @Write final WriteStart writeStart,
- @WriteUpdate final WriteStart writeUpdate,
- final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit, final RollbackAction rollback,
- final MarkStart markStart, final MarkCommit markCommit,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace,
- final SerializationFig config,
- @Assisted final CollectionScope collectionScope
- ) {
+ public EntityCollectionManagerImpl( @Write final WriteStart writeStart, @WriteUpdate final WriteStart writeUpdate,
+ final WriteUniqueVerify writeVerifyUnique,
+ final WriteOptimisticVerify writeOptimisticVerify,
+ final WriteCommit writeCommit, final RollbackAction rollback,
+ final MarkStart markStart, final MarkCommit markCommit,
+ @ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
+ final Keyspace keyspace, final SerializationFig config,
+ @Assisted final CollectionScope collectionScope ) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
- MvccValidationUtils.validateCollectionScope(collectionScope);
+ MvccValidationUtils.validateCollectionScope( collectionScope );
this.writeStart = writeStart;
this.writeUpdate = writeUpdate;
@@ -145,20 +141,20 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
- public Observable<Entity> write(final Entity entity) {
+ public Observable<Entity> write( final Entity entity ) {
//do our input validation
- Preconditions.checkNotNull(entity, "Entity is required in the new stage of the mvcc write");
+ Preconditions.checkNotNull( entity, "Entity is required in the new stage of the mvcc write" );
final Id entityId = entity.getId();
- ValidationUtils.verifyIdentity(entityId);
+ ValidationUtils.verifyIdentity( entityId );
// create our observable and start the write
- final CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>(collectionScope, entity);
+ final CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner(writeData, writeStart);
+ Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeStart );
// execute all validation stages concurrently. Needs refactored when this is done.
// https://github.com/Netflix/RxJava/issues/627
@@ -166,115 +162,119 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
// writeVerifyUnique, writeOptimisticVerify );
// return the commit result.
- return observable.map(writeCommit).doOnError(rollback);
+ return observable.map( writeCommit ).doOnError( rollback );
}
@Override
- public Observable<Void> delete(final Id entityId) {
-
- Preconditions.checkNotNull(entityId, "Entity id is required in this stage");
- Preconditions.checkNotNull(entityId.getUuid(), "Entity id is required in this stage");
- Preconditions.checkNotNull(entityId.getType(), "Entity type is required in this stage");
-
- return Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId)).map(markStart)
- .doOnNext(markCommit).map(new Func1<CollectionIoEvent<MvccEntity>, Void>() {
- @Override
- public Void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
- return null;
- }
- });
+ public Observable<Void> delete( final Id entityId ) {
+
+ Preconditions.checkNotNull( entityId, "Entity id is required in this stage" );
+ Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
+ Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
+
+ return Observable.from( new CollectionIoEvent<Id>( collectionScope, entityId ) ).map( markStart )
+ .doOnNext( markCommit ).map( new Func1<CollectionIoEvent<MvccEntity>, Void>() {
+ @Override
+ public Void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
+ return null;
+ }
+ } );
}
@Override
- public Observable<Entity> load(final Id entityId) {
+ public Observable<Entity> load( final Id entityId ) {
- Preconditions.checkNotNull(entityId, "Entity id required in the load stage");
- Preconditions.checkNotNull(entityId.getUuid(), "Entity id uuid required in load stage");
- Preconditions.checkNotNull(entityId.getType(), "Entity id type required in load stage");
+ Preconditions.checkNotNull( entityId, "Entity id required in the load stage" );
+ Preconditions.checkNotNull( entityId.getUuid(), "Entity id uuid required in load stage" );
+ Preconditions.checkNotNull( entityId.getType(), "Entity id type required in load stage" );
- return load(Collections.singleton(entityId)).map(new Func1<EntitySet, Entity>() {
+ return load( Collections.singleton( entityId ) ).map( new Func1<EntitySet, Entity>() {
@Override
- public Entity call(final EntitySet entitySet) {
- final MvccEntity entity = entitySet.getEntity(entityId);
+ public Entity call( final EntitySet entitySet ) {
+ final MvccEntity entity = entitySet.getEntity( entityId );
- if (entity == null) {
+ if ( entity == null ) {
return null;
}
return entity.getEntity().orNull();
}
- });
+ } );
}
@Override
- public Observable<EntitySet> load(final Collection<Id> entityIds) {
+ public Observable<EntitySet> load( final Collection<Id> entityIds ) {
- Preconditions.checkNotNull(entityIds, "entityIds cannot be null");
+ Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
- return Observable.create(new Observable.OnSubscribe<EntitySet>() {
+ return Observable.create( new Observable.OnSubscribe<EntitySet>() {
@Override
- public void call(final Subscriber<? super EntitySet> subscriber) {
+ public void call( final Subscriber<? super EntitySet> subscriber ) {
try {
- final EntitySet results = entitySerializationStrategy.load(
- collectionScope, entityIds, UUIDGenerator.newTimeUUID());
+ final EntitySet results =
+ entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
- subscriber.onNext(results);
+ subscriber.onNext( results );
subscriber.onCompleted();
- } catch (Exception e) {
- subscriber.onError(e);
+ }
+ catch ( Exception e ) {
+ subscriber.onError( e );
}
}
- });
+ } );
}
+
@Override
- public Observable<Id> getIdField(final Field field) {
- final List<Field> fields = Collections.singletonList(field);
- return rx.Observable.from(fields).map(new Func1<Field, Id>() {
+ public Observable<Id> getIdField( final Field field ) {
+ final List<Field> fields = Collections.singletonList( field );
+ return rx.Observable.from( fields ).map( new Func1<Field, Id>() {
@Override
- public Id call(Field field) {
+ public Id call( Field field ) {
try {
- UniqueValueSet set = uniqueValueSerializationStrategy.load(collectionScope, fields);
- UniqueValue value = set.getValue(field.getName());
+ UniqueValueSet set = uniqueValueSerializationStrategy.load( collectionScope, fields );
+ UniqueValue value = set.getValue( field.getName() );
Id id = value == null ? null : value.getEntityId();
return id;
- } catch (ConnectionException e) {
- logger.error("Failed to getIdField", e);
- throw new RuntimeException(e);
+ }
+ catch ( ConnectionException e ) {
+ logger.error( "Failed to getIdField", e );
+ throw new RuntimeException( e );
}
}
- });
+ } );
}
+
@Override
- public Observable<Entity> update(final Entity entity) {
+ public Observable<Entity> update( final Entity entity ) {
- logger.debug("Starting update process");
+ logger.debug( "Starting update process" );
//do our input validation
- Preconditions.checkNotNull(entity, "Entity is required in the new stage of the mvcc write");
+ Preconditions.checkNotNull( entity, "Entity is required in the new stage of the mvcc write" );
final Id entityId = entity.getId();
- ValidationUtils.verifyIdentity(entityId);
+ ValidationUtils.verifyIdentity( entityId );
// create our observable and start the write
- CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>(collectionScope, entity);
+ CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner(writeData, writeUpdate);
+ Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeUpdate );
- return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
+ return observable.map( writeCommit ).doOnNext( new Action1<Entity>() {
@Override
- public void call(final Entity entity) {
- logger.debug("sending entity to the queue");
+ public void call( final Entity entity ) {
+ logger.debug( "sending entity to the queue" );
//we an update, signal the fix
@@ -284,57 +284,56 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
}
- }).doOnError(rollback);
+ } ).doOnError( rollback );
}
// fire the stages
- public Observable<CollectionIoEvent<MvccEntity>> stageRunner(CollectionIoEvent<Entity> writeData,
- WriteStart writeState) {
+ public Observable<CollectionIoEvent<MvccEntity>> stageRunner( CollectionIoEvent<Entity> writeData,
+ WriteStart writeState ) {
- return Observable.from(writeData).map(writeState).doOnNext(
- new Action1<CollectionIoEvent<MvccEntity>>() {
+ return Observable.from( writeData ).map( writeState ).doOnNext( new Action1<CollectionIoEvent<MvccEntity>>() {
- @Override
- public void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
+ @Override
+ public void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
- Observable<CollectionIoEvent<MvccEntity>> unique =
- Observable.from(mvccEntityCollectionIoEvent).subscribeOn(Schedulers.io())
- .doOnNext(writeVerifyUnique);
+ Observable<CollectionIoEvent<MvccEntity>> unique =
+ Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
+ .doOnNext( writeVerifyUnique );
- // optimistic verification
- Observable<CollectionIoEvent<MvccEntity>> optimistic =
- Observable.from(mvccEntityCollectionIoEvent).subscribeOn(Schedulers.io())
- .doOnNext(writeOptimisticVerify);
+ // optimistic verification
+ Observable<CollectionIoEvent<MvccEntity>> optimistic =
+ Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
+ .doOnNext( writeOptimisticVerify );
- //wait for both to finish
- Observable.merge(unique, optimistic).toBlocking().last();
- }
- });
+ //wait for both to finish
+ Observable.merge( unique, optimistic ).toBlocking().last();
+ }
+ } );
}
@Override
- public Observable<VersionSet> getLatestVersion(final Collection<Id> entityIds) {
+ public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
- return Observable.create(new Observable.OnSubscribe<VersionSet>() {
+ return Observable.create( new Observable.OnSubscribe<VersionSet>() {
@Override
- public void call(final Subscriber<? super VersionSet> subscriber) {
+ public void call( final Subscriber<? super VersionSet> subscriber ) {
try {
final VersionSet logEntries = mvccLogEntrySerializationStrategy
- .load(collectionScope, entityIds, UUIDGenerator.newTimeUUID());
+ .load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
- subscriber.onNext(logEntries);
+ subscriber.onNext( logEntries );
subscriber.onCompleted();
-
- } catch (Exception e) {
- subscriber.onError(e);
+ }
+ catch ( Exception e ) {
+ subscriber.onError( e );
}
}
- });
+ } );
}
@@ -342,25 +341,21 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
public Health getHealth() {
try {
- ColumnFamily<String, String> CF_SYSTEM_LOCAL = new ColumnFamily<String, String>(
- "system.local",
- StringSerializer.get(),
- StringSerializer.get(),
- StringSerializer.get());
+ ColumnFamily<String, String> CF_SYSTEM_LOCAL =
+ new ColumnFamily<String, String>( "system.local", StringSerializer.get(), StringSerializer.get(),
+ StringSerializer.get() );
- OperationResult<CqlResult<String, String>> result = keyspace.prepareQuery(CF_SYSTEM_LOCAL)
- .withCql("SELECT now() FROM system.local;")
- .execute();
+ OperationResult<CqlResult<String, String>> result =
+ keyspace.prepareQuery( CF_SYSTEM_LOCAL ).withCql( "SELECT now() FROM system.local;" ).execute();
if ( result.getResult().getRows().size() == 1 ) {
return Health.GREEN;
}
-
- } catch ( ConnectionException ex ) {
- logger.error("Error connecting to Cassandra", ex);
+ }
+ catch ( ConnectionException ex ) {
+ logger.error( "Error connecting to Cassandra", ex );
}
return Health.RED;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
index b9277eb..93288af 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.entity.Id;
import com.netflix.astyanax.MutationBatch;
@@ -34,7 +35,7 @@ import com.netflix.astyanax.MutationBatch;
/**
* The interface that allows us to serialize an entity to disk
*/
-public interface MvccEntitySerializationStrategy {
+public interface MvccEntitySerializationStrategy extends Migration {
/**
* Serialize the entity to the data store with the given collection context
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java
index 927a60c..4baef84 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.entity.Id;
import com.netflix.astyanax.MutationBatch;
@@ -34,7 +35,7 @@ import com.netflix.astyanax.MutationBatch;
/**
* The interface that allows us to serialize a log entry to disk
*/
-public interface MvccLogEntrySerializationStrategy {
+public interface MvccLogEntrySerializationStrategy extends Migration {
/**
* Serialize the entity to the data store with the given collection context
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 9e2d52c..5bcb9f8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.collection.serialization.SerializationFig
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -72,7 +73,7 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
@Inject
public MarkCommit( final MvccLogEntrySerializationStrategy logStrat,
- final MvccEntitySerializationStrategy entityStrat,
+ @ProxyImpl final MvccEntitySerializationStrategy entityStrat,
final UniqueValueSerializationStrategy uniqueValueStrat, final SerializationFig serializationFig,
final Keyspace keyspace ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index 6a17197..d3c8193 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@ -37,6 +37,7 @@ import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -70,7 +71,7 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Entity>
@Inject
public WriteCommit( final MvccLogEntrySerializationStrategy logStrat,
- final MvccEntitySerializationStrategy entryStrat,
+ @ProxyImpl final MvccEntitySerializationStrategy entryStrat,
final UniqueValueSerializationStrategy uniqueValueStrat) {
Preconditions.checkNotNull( logStrat, "MvccLogEntrySerializationStrategy is required" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 4ceb407..030d9d1 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@ -23,13 +23,14 @@ import java.util.Collection;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.field.Field;
/**
* Reads and writes to UniqueValues column family.
*/
-public interface UniqueValueSerializationStrategy {
+public interface UniqueValueSerializationStrategy extends Migration {
/**
* Write the specified UniqueValue to Cassandra with optional timeToLive in milliseconds.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index bbaeb4a..1ec027f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -18,9 +18,7 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
@@ -54,21 +52,15 @@ import org.apache.usergrid.persistence.core.migration.schema.Migration;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
-import com.google.inject.Singleton;
import com.netflix.astyanax.ColumnListMutation;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.ColumnList;
-import com.netflix.astyanax.model.CompositeBuilder;
-import com.netflix.astyanax.model.CompositeParser;
-import com.netflix.astyanax.model.Composites;
import com.netflix.astyanax.model.Row;
import com.netflix.astyanax.query.RowQuery;
import com.netflix.astyanax.serializers.AbstractSerializer;
@@ -80,18 +72,13 @@ import com.netflix.astyanax.serializers.UUIDSerializer;
/**
* @author tnine
*/
-@Singleton
-public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializationStrategy, Migration {
+public abstract class MvccEntitySerializationStrategyImpl implements MvccEntitySerializationStrategy {
private static final Logger log = LoggerFactory.getLogger( MvccLogEntrySerializationStrategyImpl.class );
- private static final EntitySerializer ENTITY_JSON_SER = new EntitySerializer();
- private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
-
- private static final ByteBufferSerializer BUFFER_SERIALIZER = ByteBufferSerializer.get();
- private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
private static final CollectionScopedRowKeySerializer<Id> ROW_KEY_SER =
@@ -104,6 +91,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
protected final Keyspace keyspace;
protected final SerializationFig serializationFig;
protected final EntityRepair repair;
+ private final AbstractSerializer<EntityWrapper> entityJsonSerializer;
@Inject
@@ -111,6 +99,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
this.keyspace = keyspace;
this.serializationFig = serializationFig;
this.repair = new EntityRepairImpl( this, serializationFig );
+ this.entityJsonSerializer = getEntitySerializer();
}
@@ -126,7 +115,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
@Override
public void doOp( final ColumnListMutation<UUID> colMutation ) {
try {
- colMutation.putColumn( colName, ENTITY_JSON_SER
+ colMutation.putColumn( colName, entityJsonSerializer
.toByteBuffer( new EntityWrapper( entity.getStatus(), entity.getEntity() ) ) );
}
catch ( Exception e ) {
@@ -207,7 +196,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
final Column<UUID> column = columns.getColumnByIndex( 0 );
- final MvccEntity parsedEntity = new MvccColumnParser( entityId ).parseColumn( column );
+ final MvccEntity parsedEntity = new MvccColumnParser( entityId, entityJsonSerializer ).parseColumn( column );
//we *might* need to repair, it's not clear so check before loading into result sets
final MvccEntity maybeRepaired = repair.maybeRepair( collectionScope, parsedEntity );
@@ -245,7 +234,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( rowKey )
.withColumnRange( version, null, false, fetchSize );
- return new ColumnNameIterator( query, new MvccColumnParser( entityId ), false );
+ return new ColumnNameIterator( query, new MvccColumnParser( entityId, entityJsonSerializer ), false );
}
@@ -275,7 +264,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( rowKey )
.withColumnRange( null, version, true, fetchSize );
- return new ColumnNameIterator( query, new MvccColumnParser( entityId ), false );
+ return new ColumnNameIterator( query, new MvccColumnParser( entityId, entityJsonSerializer ), false );
}
@@ -290,7 +279,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
return doWrite( collectionScope, entityId, new RowOp() {
@Override
public void doOp( final ColumnListMutation<UUID> colMutation ) {
- colMutation.putColumn( version, ENTITY_JSON_SER
+ colMutation.putColumn( version, entityJsonSerializer
.toByteBuffer( new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() ) ) );
}
} );
@@ -367,12 +356,12 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
/**
* Simple bean wrapper for state and entity
*/
- private static class EntityWrapper {
- private final MvccEntity.Status status;
- private final Optional<Entity> entity;
+ protected static class EntityWrapper {
+ protected final MvccEntity.Status status;
+ protected final Optional<Entity> entity;
- private EntityWrapper( final MvccEntity.Status status, final Optional<Entity> entity ) {
+ protected EntityWrapper( final MvccEntity.Status status, final Optional<Entity> entity ) {
this.status = status;
this.entity = entity;
}
@@ -385,10 +374,12 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
private static final class MvccColumnParser implements ColumnParser<UUID, MvccEntity> {
private final Id id;
+ private final AbstractSerializer<EntityWrapper> entityJsonSerializer;
- private MvccColumnParser( Id id ) {
+ private MvccColumnParser( Id id, final AbstractSerializer<EntityWrapper> entityJsonSerializer ) {
this.id = id;
+ this.entityJsonSerializer = entityJsonSerializer;
}
@@ -399,7 +390,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
final UUID version = column.getName();
try {
- deSerialized = column.getValue( ENTITY_JSON_SER );
+ deSerialized = column.getValue( entityJsonSerializer );
}
catch ( DataCorruptionException e ) {
log.error(
@@ -421,128 +412,9 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
}
- public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
-
-
- public static final SmileFactory f = new SmileFactory();
-
- public static ObjectMapper mapper;
-
- private static byte[] STATE_COMPLETE = new byte[] { 0 };
- private static byte[] STATE_DELETED = new byte[] { 1 };
- private static byte[] STATE_PARTIAL = new byte[] { 2 };
-
- private static byte[] VERSION = new byte[] { 0 };
-
-
-
- public EntitySerializer() {
- try {
- mapper = new ObjectMapper( f );
- // mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
- // causes slowness
- mapper.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Error setting up mapper", e );
- }
- }
-
-
- @Override
- public ByteBuffer toByteBuffer( final EntityWrapper wrapper ) {
- if ( wrapper == null ) {
- return null;
- }
-
- CompositeBuilder builder = Composites.newCompositeBuilder();
-
- builder.addBytes( VERSION );
-
- //mark this version as empty
- if ( !wrapper.entity.isPresent() ) {
- //we're empty
- builder.addBytes( STATE_DELETED );
-
- return builder.build();
- }
-
- //we have an entity
-
- if ( wrapper.status == MvccEntity.Status.COMPLETE ) {
- builder.addBytes( STATE_COMPLETE );
- }
-
- else {
- builder.addBytes( STATE_PARTIAL );
- }
-
- try {
- final byte[] entityBytes = mapper.writeValueAsBytes( wrapper.entity.get() ) ;
- builder.addBytes( entityBytes );
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Unable to serialize entity", e );
- }
-
- return builder.build();
- }
-
-
- @Override
- public EntityWrapper fromByteBuffer( final ByteBuffer byteBuffer ) {
-
- /**
- * We intentionally turn data corruption exceptions when we're unable to de-serialize
- * the data in cassandra. If this occurs, we'll never be able to de-serialize it
- * and it should be considered lost. This is an error that is occuring due to a bug
- * in serializing the entity. This is a lazy recognition + repair signal for deployment with
- * existing systems.
- */
- CompositeParser parser;
- try {
- parser = Composites.newCompositeParser( byteBuffer );
- }
- catch ( Exception e ) {
- throw new DataCorruptionException("Unable to de-serialze entity", e);
- }
-
- byte[] version = parser.read( BYTES_ARRAY_SERIALIZER );
-
- if ( !Arrays.equals( VERSION, version ) ) {
- throw new UnsupportedOperationException( "A version of type " + version + " is unsupported" );
- }
-
- byte[] state = parser.read( BYTES_ARRAY_SERIALIZER );
-
- // it's been deleted, remove it
-
- if ( Arrays.equals( STATE_DELETED, state ) ) {
- return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
- }
-
- Entity storedEntity;
-
- ByteBuffer jsonBytes = parser.read( BUFFER_SERIALIZER );
- byte[] array = jsonBytes.array();
- int start = jsonBytes.arrayOffset();
- int length = jsonBytes.remaining();
-
- try {
- storedEntity = mapper.readValue( array, start, length, Entity.class );
- }
- catch ( Exception e ) {
- throw new DataCorruptionException( "Unable to read entity data", e );
- }
-
- final Optional<Entity> entity = Optional.of( storedEntity );
-
- if ( Arrays.equals( STATE_COMPLETE, state ) ) {
- return new EntityWrapper( MvccEntity.Status.COMPLETE, entity );
- }
-
- // it's partial by default
- return new EntityWrapper( MvccEntity.Status.PARTIAL, entity );
- }
- }
+ /**
+ * Return the entity serializer for this instance
+ * @return
+ */
+ protected abstract AbstractSerializer<EntityWrapper> getEntitySerializer();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
new file mode 100644
index 0000000..a9e01b1
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
+import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.PreviousImpl;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+
+
+/**
+ * Version 3 implementation of entity serialization. This will proxy writes and reads so that during
+ * migration data goes to both sources and is read from the old source. After the ugprade completes,
+ * it will be available from the new source
+ */
+@Singleton
+public class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySerializationStrategy {
+
+
+ public static final int MIGRATION_VERSION = 3;
+
+ private final DataMigrationManager dataMigrationManager;
+ private final Keyspace keyspace;
+ private final MvccEntitySerializationStrategy previous;
+ private final MvccEntitySerializationStrategy current;
+
+
+ @Inject
+ public MvccEntitySerializationStrategyProxyImpl( final DataMigrationManager dataMigrationManager,
+ final Keyspace keyspace,
+ @PreviousImpl final MvccEntitySerializationStrategy previous,
+ @CurrentImpl final MvccEntitySerializationStrategy current ) {
+ this.dataMigrationManager = dataMigrationManager;
+ this.keyspace = keyspace;
+ this.previous = previous;
+ this.current = current;
+ }
+
+
+ @Override
+ public MutationBatch write( final CollectionScope context, final MvccEntity entity ) {
+ if ( isOldVersion() ) {
+ final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
+
+ aggregateBatch.mergeShallow( previous.write( context, entity ) );
+ aggregateBatch.mergeShallow( current.write( context, entity ) );
+
+ return aggregateBatch;
+ }
+
+ return current.write( context, entity );
+ }
+
+
+ @Override
+ public EntitySet load( final CollectionScope scope, final Collection<Id> entityIds, final UUID maxVersion ) {
+ if ( isOldVersion() ) {
+ return previous.load( scope, entityIds, maxVersion );
+ }
+
+ return current.load( scope, entityIds, maxVersion );
+ }
+
+
+ @Override
+ public Iterator<MvccEntity> load( final CollectionScope context, final Id entityId, final UUID version,
+ final int fetchSize ) {
+ if ( isOldVersion() ) {
+ return previous.load( context, entityId, version, fetchSize );
+ }
+
+ return current.load( context, entityId, version, fetchSize );
+ }
+
+
+ @Override
+ public Iterator<MvccEntity> loadHistory( final CollectionScope context, final Id entityId, final UUID version,
+ final int fetchSize ) {
+ if ( isOldVersion() ) {
+ return previous.loadHistory( context, entityId, version, fetchSize );
+ }
+
+ return current.loadHistory( context, entityId, version, fetchSize );
+ }
+
+
+ @Override
+ public MutationBatch mark( final CollectionScope context, final Id entityId, final UUID version ) {
+ if ( isOldVersion() ) {
+ final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
+
+ aggregateBatch.mergeShallow( previous.mark( context, entityId, version ) );
+ aggregateBatch.mergeShallow( current.mark( context, entityId, version ) );
+
+ return aggregateBatch;
+ }
+
+ return current.mark( context, entityId, version );
+ }
+
+
+ @Override
+ public MutationBatch delete( final CollectionScope context, final Id entityId, final UUID version ) {
+ if ( isOldVersion() ) {
+ final MutationBatch aggregateBatch = keyspace.prepareMutationBatch();
+
+ aggregateBatch.mergeShallow( previous.delete( context, entityId, version ) );
+ aggregateBatch.mergeShallow( current.delete( context, entityId, version ) );
+
+ return aggregateBatch;
+ }
+
+ return current.delete( context, entityId, version );
+ }
+
+
+ /**
+ * Return true if we're on an old version
+ */
+ private boolean isOldVersion() {
+ return dataMigrationManager.getCurrentVersion() < MIGRATION_VERSION;
+ }
+
+
+ @Override
+ public Collection<MultiTennantColumnFamilyDefinition> getColumnFamilies() {
+ return Collections.emptyList();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
new file mode 100644
index 0000000..49b3486
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.model.CompositeBuilder;
+import com.netflix.astyanax.model.CompositeParser;
+import com.netflix.astyanax.model.Composites;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+import com.netflix.astyanax.serializers.ByteBufferSerializer;
+import com.netflix.astyanax.serializers.BytesArraySerializer;
+
+
+/**
+ * Version 1 implementation of entity serialization
+ */
+@Singleton
+public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializationStrategyImpl {
+
+ private static final EntitySerializer ENTITY_JSON_SER = new EntitySerializer();
+
+
+ @Inject
+ public MvccEntitySerializationStrategyV1Impl( final Keyspace keyspace, final SerializationFig serializationFig ) {
+ super( keyspace, serializationFig );
+ }
+
+
+ @Override
+ protected AbstractSerializer<MvccEntitySerializationStrategyImpl.EntityWrapper> getEntitySerializer() {
+ return ENTITY_JSON_SER;
+ }
+
+
+ public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
+
+
+ private static final ByteBufferSerializer BUFFER_SERIALIZER = ByteBufferSerializer.get();
+
+ private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+
+
+ public static final SmileFactory f = new SmileFactory();
+
+ public static ObjectMapper mapper;
+
+ private static byte[] STATE_COMPLETE = new byte[] { 0 };
+ private static byte[] STATE_DELETED = new byte[] { 1 };
+ private static byte[] STATE_PARTIAL = new byte[] { 2 };
+
+ private static byte[] VERSION = new byte[] { 0 };
+
+
+ public EntitySerializer() {
+ try {
+ mapper = new ObjectMapper( f );
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
+ // causes slowness
+ mapper.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Error setting up mapper", e );
+ }
+ }
+
+
+ @Override
+ public ByteBuffer toByteBuffer( final EntityWrapper wrapper ) {
+ if ( wrapper == null ) {
+ return null;
+ }
+
+ CompositeBuilder builder = Composites.newCompositeBuilder();
+
+ builder.addBytes( VERSION );
+
+ //mark this version as empty
+ if ( !wrapper.entity.isPresent() ) {
+ //we're empty
+ builder.addBytes( STATE_DELETED );
+
+ return builder.build();
+ }
+
+ //we have an entity
+
+ if ( wrapper.status == MvccEntity.Status.COMPLETE ) {
+ builder.addBytes( STATE_COMPLETE );
+ }
+
+ else {
+ builder.addBytes( STATE_PARTIAL );
+ }
+
+ try {
+ final byte[] entityBytes = mapper.writeValueAsBytes( wrapper.entity.get() );
+ builder.addBytes( entityBytes );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to serialize entity", e );
+ }
+
+ return builder.build();
+ }
+
+
+ @Override
+ public EntityWrapper fromByteBuffer( final ByteBuffer byteBuffer ) {
+
+ /**
+ * We intentionally turn data corruption exceptions when we're unable to de-serialize
+ * the data in cassandra. If this occurs, we'll never be able to de-serialize it
+ * and it should be considered lost. This is an error that is occuring due to a bug
+ * in serializing the entity. This is a lazy recognition + repair signal for deployment with
+ * existing systems.
+ */
+ CompositeParser parser;
+ try {
+ parser = Composites.newCompositeParser( byteBuffer );
+ }
+ catch ( Exception e ) {
+ throw new DataCorruptionException( "Unable to de-serialze entity", e );
+ }
+
+ byte[] version = parser.read( BYTES_ARRAY_SERIALIZER );
+
+ if ( !Arrays.equals( VERSION, version ) ) {
+ throw new UnsupportedOperationException( "A version of type " + version + " is unsupported" );
+ }
+
+ byte[] state = parser.read( BYTES_ARRAY_SERIALIZER );
+
+ // it's been deleted, remove it
+
+ if ( Arrays.equals( STATE_DELETED, state ) ) {
+ return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
+ }
+
+ Entity storedEntity;
+
+ ByteBuffer jsonBytes = parser.read( BUFFER_SERIALIZER );
+ byte[] array = jsonBytes.array();
+ int start = jsonBytes.arrayOffset();
+ int length = jsonBytes.remaining();
+
+ try {
+ storedEntity = mapper.readValue( array, start, length, Entity.class );
+ }
+ catch ( Exception e ) {
+ throw new DataCorruptionException( "Unable to read entity data", e );
+ }
+
+ final Optional<Entity> entity = Optional.of( storedEntity );
+
+ if ( Arrays.equals( STATE_COMPLETE, state ) ) {
+ return new EntityWrapper( MvccEntity.Status.COMPLETE, entity );
+ }
+
+ // it's partial by default
+ return new EntityWrapper( MvccEntity.Status.PARTIAL, entity );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
new file mode 100644
index 0000000..f80960d
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.model.CompositeBuilder;
+import com.netflix.astyanax.model.CompositeParser;
+import com.netflix.astyanax.model.Composites;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+import com.netflix.astyanax.serializers.ByteBufferSerializer;
+import com.netflix.astyanax.serializers.BytesArraySerializer;
+
+
+/**
+ * Version 1 implementation of entity serialization
+ */
+@Singleton
+public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializationStrategyImpl {
+
+ private static final EntitySerializer ENTITY_JSON_SER = new EntitySerializer();
+
+
+ @Inject
+ public MvccEntitySerializationStrategyV2Impl( final Keyspace keyspace, final SerializationFig serializationFig ) {
+ super( keyspace, serializationFig );
+ }
+
+
+ @Override
+ protected AbstractSerializer<EntityWrapper> getEntitySerializer() {
+ return ENTITY_JSON_SER;
+ }
+
+
+ public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
+
+
+ private static final ByteBufferSerializer BUFFER_SERIALIZER = ByteBufferSerializer.get();
+
+ private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+
+
+ public static final SmileFactory f = new SmileFactory();
+
+ public static ObjectMapper mapper;
+
+ private static byte[] STATE_COMPLETE = new byte[] { 0 };
+ private static byte[] STATE_DELETED = new byte[] { 1 };
+ private static byte[] STATE_PARTIAL = new byte[] { 2 };
+
+ private static byte[] VERSION = new byte[] { 0 };
+
+
+ public EntitySerializer() {
+ try {
+ mapper = new ObjectMapper( f );
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
+ // causes slowness
+ mapper.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Error setting up mapper", e );
+ }
+ }
+
+
+ @Override
+ public ByteBuffer toByteBuffer( final EntityWrapper wrapper ) {
+ if ( wrapper == null ) {
+ return null;
+ }
+
+ CompositeBuilder builder = Composites.newCompositeBuilder();
+
+ builder.addBytes( VERSION );
+
+ //mark this version as empty
+ if ( !wrapper.entity.isPresent() ) {
+ //we're empty
+ builder.addBytes( STATE_DELETED );
+
+ return builder.build();
+ }
+
+ //we have an entity
+
+ if ( wrapper.status == MvccEntity.Status.COMPLETE ) {
+ builder.addBytes( STATE_COMPLETE );
+ }
+
+ else {
+ builder.addBytes( STATE_PARTIAL );
+ }
+
+ try {
+ final byte[] entityBytes = mapper.writeValueAsBytes( wrapper.entity.get() );
+ builder.addBytes( entityBytes );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to serialize entity", e );
+ }
+
+ return builder.build();
+ }
+
+
+ @Override
+ public EntityWrapper fromByteBuffer( final ByteBuffer byteBuffer ) {
+
+ /**
+ * We intentionally turn data corruption exceptions when we're unable to de-serialize
+ * the data in cassandra. If this occurs, we'll never be able to de-serialize it
+ * and it should be considered lost. This is an error that is occuring due to a bug
+ * in serializing the entity. This is a lazy recognition + repair signal for deployment with
+ * existing systems.
+ */
+ CompositeParser parser;
+ try {
+ parser = Composites.newCompositeParser( byteBuffer );
+ }
+ catch ( Exception e ) {
+ throw new DataCorruptionException( "Unable to de-serialze entity", e );
+ }
+
+ byte[] version = parser.read( BYTES_ARRAY_SERIALIZER );
+
+ if ( !Arrays.equals( VERSION, version ) ) {
+ throw new UnsupportedOperationException( "A version of type " + version + " is unsupported" );
+ }
+
+ byte[] state = parser.read( BYTES_ARRAY_SERIALIZER );
+
+ // it's been deleted, remove it
+
+ if ( Arrays.equals( STATE_DELETED, state ) ) {
+ return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
+ }
+
+ Entity storedEntity;
+
+ ByteBuffer jsonBytes = parser.read( BUFFER_SERIALIZER );
+ byte[] array = jsonBytes.array();
+ int start = jsonBytes.arrayOffset();
+ int length = jsonBytes.remaining();
+
+ try {
+ storedEntity = mapper.readValue( array, start, length, Entity.class );
+ }
+ catch ( Exception e ) {
+ throw new DataCorruptionException( "Unable to read entity data", e );
+ }
+
+ final Optional<Entity> entity = Optional.of( storedEntity );
+
+ if ( Arrays.equals( STATE_COMPLETE, state ) ) {
+ return new EntityWrapper( MvccEntity.Status.COMPLETE, entity );
+ }
+
+ // it's partial by default
+ return new EntityWrapper( MvccEntity.Status.PARTIAL, entity );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
index 7368e9a..831091d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
@@ -71,7 +71,7 @@ import com.netflix.astyanax.serializers.UUIDSerializer;
* @author tnine
*/
@Singleton
-public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerializationStrategy, Migration {
+public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerializationStrategy {
private static final Logger LOG = LoggerFactory.getLogger( MvccLogEntrySerializationStrategyImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index 8a4a9ac..eb9c374 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -21,9 +21,13 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.PreviousImpl;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.migration.schema.Migration;
import com.google.inject.AbstractModule;
+import com.google.inject.Key;
import com.google.inject.multibindings.Multibinder;
@@ -37,14 +41,23 @@ public class SerializationModule extends AbstractModule {
// bind the serialization strategies
- bind( MvccEntitySerializationStrategy.class ).to( MvccEntitySerializationStrategyImpl.class );
+
+ //We've migrated this one, so we need to set up the previous, current, and proxy
+ bind( MvccEntitySerializationStrategy.class ).annotatedWith( PreviousImpl.class )
+ .to( MvccEntitySerializationStrategyV1Impl.class );
+ bind( MvccEntitySerializationStrategy.class ).annotatedWith( CurrentImpl.class )
+ .to( MvccEntitySerializationStrategyV2Impl.class );
+ bind( MvccEntitySerializationStrategy.class ).annotatedWith( ProxyImpl.class )
+ .to( MvccEntitySerializationStrategyProxyImpl.class );
+
bind( MvccLogEntrySerializationStrategy.class ).to( MvccLogEntrySerializationStrategyImpl.class );
bind( UniqueValueSerializationStrategy.class ).to( UniqueValueSerializationStrategyImpl.class );
//do multibindings for migrations
Multibinder<Migration> uriBinder = Multibinder.newSetBinder( binder(), Migration.class );
- uriBinder.addBinding().to( MvccEntitySerializationStrategyImpl.class );
- uriBinder.addBinding().to( MvccLogEntrySerializationStrategyImpl.class );
- uriBinder.addBinding().to( UniqueValueSerializationStrategyImpl.class );
+ uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategy.class, PreviousImpl.class ) );
+ uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategy.class, CurrentImpl.class ) );
+ uriBinder.addBinding().to( Key.get( MvccLogEntrySerializationStrategy.class ) );
+ uriBinder.addBinding().to( Key.get( UniqueValueSerializationStrategy.class ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index b6b3cab..be95b08 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -57,7 +57,7 @@ import com.netflix.astyanax.util.RangeBuilder;
/**
* Reads and writes to UniqueValues column family.
*/
-public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializationStrategy, Migration {
+public class UniqueValueSerializationStrategyImpl implements UniqueValueSerializationStrategy {
private static final Logger log = LoggerFactory.getLogger( UniqueValueSerializationStrategyImpl.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerFactoryTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerFactoryTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerFactoryTest.java
index f2deb3e..0d44e98 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerFactoryTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerFactoryTest.java
@@ -6,7 +6,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 66f162f..2d18675 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -23,18 +23,19 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import org.apache.usergrid.persistence.core.test.UseModules;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
@@ -42,10 +43,10 @@ import org.apache.usergrid.persistence.model.field.BooleanField;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.field.IntegerField;
import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.fasterxml.uuid.UUIDComparator;
import com.google.inject.Inject;
-import org.apache.usergrid.persistence.core.util.Health;
import rx.Observable;
@@ -58,8 +59,8 @@ import static org.junit.Assert.fail;
/** @author tnine */
-@RunWith(ITRunner.class)
-@UseModules(TestCollectionModule.class)
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
public class EntityCollectionManagerIT {
@Inject
private EntityCollectionManagerFactory factory;
@@ -533,7 +534,7 @@ public class EntityCollectionManagerIT {
}
- @Test(expected = IllegalArgumentException.class)
+ @Test( expected = IllegalArgumentException.class )
public void readTooLarge() {
final CollectionScope context =
@@ -630,8 +631,7 @@ public class EntityCollectionManagerIT {
final UUID v2Version = v2Created.getVersion();
- assertTrue( "Newer version in v2",
- UUIDComparator.staticCompare( v2Version, v1Version) > 0 );
+ assertTrue( "Newer version in v2", UUIDComparator.staticCompare( v2Version, v1Version ) > 0 );
final VersionSet resultsV2 = manager.getLatestVersion( Arrays.asList( v1Created.getId() ) ).toBlocking().last();
@@ -678,12 +678,9 @@ public class EntityCollectionManagerIT {
final UUID v2Version = v2Created.getVersion();
-
-
assertEquals( "Same entityId", v1Created.getId(), v2Created.getId() );
- assertTrue( "Newer version in v2",
- UUIDComparator.staticCompare( v2Version, v1Version ) > 0 );
+ assertTrue( "Newer version in v2", UUIDComparator.staticCompare( v2Version, v1Version ) > 0 );
final VersionSet resultsV2 = manager.getLatestVersion( Arrays.asList( v1Created.getId() ) ).toBlocking().last();
@@ -700,11 +697,60 @@ public class EntityCollectionManagerIT {
@Test
public void healthTest() {
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
final EntityCollectionManager manager = factory.createCollectionManager( context );
assertEquals( Health.GREEN, manager.getHealth() );
}
+
+
+ /**
+ * Tests an entity with more than 65535 bytes worth of data
+ */
+ @Test
+ public void largeEntityWriteRead() {
+ final int setSize = 65535 * 2;
+
+ int currentLength = 0;
+
+ final Entity entity = new Entity( new SimpleId( "test" ) );
+
+ //generate a really large string value
+ StringBuilder builder = new StringBuilder();
+
+ for ( int i = 0; i < 100; i++ ) {
+ builder.append( UUIDGenerator.newTimeUUID().toString() );
+ }
+
+ final String value = builder.toString();
+
+
+ //loop until our size is beyond the set size
+ for ( int i = 0; currentLength < setSize; i++ ) {
+ final String key = "newStringField" + i;
+
+ entity.setField( new StringField( key, value ) );
+
+ currentLength += key.length() + value.length();
+ }
+
+
+ //now we have one massive, entity, save it and retrieve it.
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ final Entity saved = manager.write( entity ).toBlocking().last();
+
+
+ assertEquals( entity, saved );
+
+ //now load it
+ final Entity loaded = manager.load( entity.getId() ).toBlocking().last();
+
+ assertEquals( entity, loaded );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
index d4f4d81..11ad389 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
@@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory;
import org.apache.commons.lang3.time.StopWatch;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerSyncIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerSyncIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerSyncIT.java
index ec35ed2..d4881de 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerSyncIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerSyncIT.java
@@ -6,7 +6,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2bd1c950/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/MigrationManagerRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/MigrationManagerRule.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/MigrationManagerRule.java
deleted file mode 100644
index 6b02b63..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/MigrationManagerRule.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.apache.usergrid.persistence.collection.guice;
-
-
-import org.junit.rules.ExternalResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
-
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-
-/**
- */
-@Singleton
-public class MigrationManagerRule extends ExternalResource {
- private static final Logger LOG = LoggerFactory.getLogger( MigrationManagerRule.class );
-
- private MigrationManager migrationManager;
-
-
- @Inject
- public void setMigrationManager( MigrationManager migrationManager ) {
- this.migrationManager = migrationManager;
- }
-
-
- @Override
- protected void before() throws MigrationException {
- LOG.info( "Starting migration" );
-
- migrationManager.migrate();
-
- LOG.info( "Migration complete" );
- }
-}
[16/19] incubator-usergrid git commit: Added value validation tests
Posted by sf...@apache.org.
Added value validation tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3e625b1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3e625b1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3e625b1e
Branch: refs/heads/two-dot-o
Commit: 3e625b1e9160c384111d39bb3a740c6bf978a4a9
Parents: fb90ae7
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 21 13:00:36 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 21 13:00:36 2014 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../corepersistence/util/CpNamingUtils.java | 4 +-
.../serialization/SerializationFig.java | 6 +-
.../serialization/impl/SerializationModule.java | 4 +
.../serialization/impl/SettingsValidation.java | 58 ++++++++
.../impl/SettingsValidationTest.java | 131 +++++++++++++++++++
6 files changed, 199 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e625b1e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index e6b8bce..b2d854b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -583,7 +583,7 @@ public class CpEntityManager implements EntityManager {
private Observable deleteAsync( EntityRef entityRef ) throws Exception {
- CollectionScope collectionScope =getCollectionScopeNameFromEntityType(applicationScope.getApplication(), entityRef.getType() );
+ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), entityRef.getType() );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e625b1e/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 684b6e0..0d7b6ff 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -79,9 +79,7 @@ public class CpNamingUtils {
* @return The collectionScope
*/
public static CollectionScope getCollectionScopeNameFromEntityType(final Id applicationId, final String type){
- return
- new CollectionScopeImpl( applicationId, applicationId,
- getCollectionScopeNameFromEntityType( type ) );
+ return new CollectionScopeImpl( applicationId, applicationId, getCollectionScopeNameFromEntityType( type ) );
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e625b1e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 0687312..9fb04d1 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -13,6 +13,8 @@ import org.safehaus.guicyfig.Key;
@FigSingleton
public interface SerializationFig extends GuicyFig {
+ String COLLECTION_MAX_ENTITY_SIZE = "collection.max.entity.size";
+
/**
* Time to live timeout in seconds.
*
@@ -58,7 +60,7 @@ public interface SerializationFig extends GuicyFig {
int getTaskPoolQueueSize();
/**
- * The maximum amount of entities we can load at any one time
+ * The maximum amount of entities we can load in a single request
*/
@Key( "collection.max.load.size" )
@Default( "100" )
@@ -71,7 +73,7 @@ public interface SerializationFig extends GuicyFig {
*
* (15mb thrift buffer * .9) / 100 (default max load size)
*/
- @Key( "collection.max.entity.size" )
+ @Key( COLLECTION_MAX_ENTITY_SIZE )
@Default( "141557" )
int getMaxEntitySize();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e625b1e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
index eb9c374..33a092d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SerializationModule.java
@@ -59,5 +59,9 @@ public class SerializationModule extends AbstractModule {
uriBinder.addBinding().to( Key.get( MvccEntitySerializationStrategy.class, CurrentImpl.class ) );
uriBinder.addBinding().to( Key.get( MvccLogEntrySerializationStrategy.class ) );
uriBinder.addBinding().to( Key.get( UniqueValueSerializationStrategy.class ) );
+
+
+ //bind our settings as an eager singleton so it's checked on startup
+ bind(SettingsValidation.class).asEagerSingleton();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e625b1e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidation.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidation.java
new file mode 100644
index 0000000..6715cfb
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidation.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
+
+import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ *
+ * This element validates our SerializationFig and CassandraFig are correct for transport values
+ * TODO, this feels like a hack (Even though it's legal in GUICE) When we have more time, we should look at using
+ * something like visitors and SPI to perform validation
+ */
+@Singleton
+public class SettingsValidation {
+
+
+ @Inject
+ public SettingsValidation( final CassandraFig cassandraFig, final SerializationFig serializationFig ) {
+ final int thriftBufferSize = cassandraFig.getThriftBufferSize();
+
+ Preconditions.checkArgument( thriftBufferSize > 0, CassandraFig.THRIFT_TRANSPORT_SIZE + " must be > than 0" );
+
+ final int usableThriftBufferSize = ( int ) (thriftBufferSize*.9);
+
+ final int maxEntitySize = serializationFig.getMaxEntitySize();
+
+ Preconditions.checkArgument( maxEntitySize > 0, CassandraFig.THRIFT_TRANSPORT_SIZE + " must be > than 0" );
+
+ Preconditions.checkArgument(usableThriftBufferSize >= maxEntitySize, "You cannot set the max entity size to more than the thrift buffer size * .9. Maximum usable thrift size is " + usableThriftBufferSize + " and max entity size is " + maxEntitySize);
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3e625b1e/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidationTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidationTest.java
new file mode 100644
index 0000000..807bc4e
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/SettingsValidationTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+/**
+ * Performs basic unit tests on our settings validation
+ */
+public class SettingsValidationTest {
+
+ /**
+ * Tests that when we're within range, it passes
+ */
+ @Test
+ public void withinBounds(){
+ CassandraFig cassandraFig = mock(CassandraFig.class);
+
+ final int thriftSize = 15728640;
+
+ final int usableThriftSize = ( int ) (thriftSize * .9);
+
+ when(cassandraFig.getThriftBufferSize()).thenReturn( thriftSize );
+
+
+ SerializationFig serializationFig = mock(SerializationFig.class);
+
+ when(serializationFig.getMaxEntitySize()).thenReturn( usableThriftSize );
+
+ new SettingsValidation( cassandraFig, serializationFig);
+
+ when(serializationFig.getMaxEntitySize()).thenReturn( usableThriftSize -1 );
+
+ new SettingsValidation( cassandraFig, serializationFig);
+
+ }
+
+
+ /**
+ * Tests that when we're within range, it passes
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void outOfBounds(){
+ CassandraFig cassandraFig = mock(CassandraFig.class);
+
+ final int thriftSize = 15728640;
+
+ final int usableThriftSize = ( int ) (thriftSize * .9);
+
+ when(cassandraFig.getThriftBufferSize()).thenReturn( thriftSize );
+
+
+ SerializationFig serializationFig = mock(SerializationFig.class);
+
+ when(serializationFig.getMaxEntitySize()).thenReturn( usableThriftSize+1 );
+
+ new SettingsValidation( cassandraFig, serializationFig);
+
+
+
+ }
+
+
+
+ /**
+ * Tests that when we're within range, it passes
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void zeroBufferSize(){
+ CassandraFig cassandraFig = mock(CassandraFig.class);
+
+ final int thriftSize = 0;
+
+ when(cassandraFig.getThriftBufferSize()).thenReturn( thriftSize );
+
+ SerializationFig serializationFig = mock(SerializationFig.class);
+
+ new SettingsValidation( cassandraFig, serializationFig);
+
+
+
+ }
+
+ /**
+ * Tests that when we're within range, it passes
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void zeroEntitySize(){
+ CassandraFig cassandraFig = mock(CassandraFig.class);
+
+ final int thriftSize = 15728640;
+
+ when(cassandraFig.getThriftBufferSize()).thenReturn( thriftSize );
+
+
+ SerializationFig serializationFig = mock(SerializationFig.class);
+
+ when(serializationFig.getMaxEntitySize()).thenReturn( 0 );
+
+ new SettingsValidation( cassandraFig, serializationFig);
+
+
+
+ }
+}
[19/19] incubator-usergrid git commit: Setting to current default of
1k. We need to refactor this setting to be common between all modules and
enforced in all modules.
Posted by sf...@apache.org.
Setting to current default of 1k. We need to refactor this setting to be common between all modules and enforced in all modules.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0a92bee7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0a92bee7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0a92bee7
Branch: refs/heads/two-dot-o
Commit: 0a92bee7988cb2a5207aef1c5dff5cd4c23c2f8a
Parents: b240c6f
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 21 14:18:21 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 21 14:18:21 2014 -0700
----------------------------------------------------------------------
.../persistence/collection/serialization/SerializationFig.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0a92bee7/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 9fb04d1..381a24e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -61,9 +61,10 @@ public interface SerializationFig extends GuicyFig {
/**
* The maximum amount of entities we can load in a single request
+ * TODO, change this and move it into a common setting that both query and collection share
*/
@Key( "collection.max.load.size" )
- @Default( "100" )
+ @Default( "1000" )
int getMaxLoadSize();
[11/19] incubator-usergrid git commit: Completed fix in collection
module. Renamed read methods so that it's clear we're advancing a buffer and
mutating state.
Posted by sf...@apache.org.
Completed fix in collection module. Renamed read methods so that it's clear we're advancing a buffer and mutating state.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/68f4e0f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/68f4e0f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/68f4e0f1
Branch: refs/heads/two-dot-o
Commit: 68f4e0f1a4942248a1f41c3c61d97fbf93440010
Parents: 262ad8b
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 20 11:25:14 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 20 11:25:14 2014 -0700
----------------------------------------------------------------------
.../usergrid/persistence/LargeEntityIT.java | 2 +
.../MvccEntitySerializationStrategyImpl.java | 34 +++----
.../MvccEntitySerializationStrategyV1Impl.java | 24 +++++
.../MvccEntitySerializationStrategyV2Impl.java | 95 +++++++++++++-------
.../collection/EntityCollectionManagerIT.java | 2 +-
...ccEntitySerializationSTrategyV2FixTests.java | 82 -----------------
...MvccEntitySerializationStrategyImplTest.java | 54 +++++++++++
...cEntitySerializationStrategyProxyV1Test.java | 25 ++++++
...cEntitySerializationStrategyProxyV2Test.java | 27 +++++-
...ccEntitySerializationStrategyV1ImplTest.java | 27 ++++++
...ccEntitySerializationStrategyV2ImplTest.java | 27 +++++-
.../collection/util/EntityHelper.java | 2 +-
.../core/astyanax/FieldBufferParser.java | 6 +-
.../astyanax/FieldBufferSerializerTest.java | 12 +--
14 files changed, 275 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
index ad6ca27..eef2820 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
@@ -25,6 +25,7 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Map;
+import org.aspectj.lang.annotation.Before;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
@@ -47,6 +48,7 @@ public class LargeEntityIT extends AbstractCoreIT {
public Application app = new CoreApplication( setup );
+
/**
* Tests creating a large entity, then loading it, modifying it, saving it, then loading it again
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index 4138f8d..4ac5d24 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -74,20 +74,11 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
private static final Logger log = LoggerFactory.getLogger( MvccLogEntrySerializationStrategyImpl.class );
- private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
-
-
- private static final CollectionScopedRowKeySerializer<Id> ROW_KEY_SER =
- new CollectionScopedRowKeySerializer<Id>( ID_SER );
-
- private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> CF_ENTITY_DATA =
- new MultiTennantColumnFamily<>( "Entity_Version_Data", ROW_KEY_SER, UUIDSerializer.get() );
-
-
protected final Keyspace keyspace;
protected final SerializationFig serializationFig;
protected final EntityRepair repair;
private final AbstractSerializer<EntityWrapper> entityJsonSerializer;
+ private final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> columnFamily;
@Inject
@@ -96,6 +87,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
this.serializationFig = serializationFig;
this.repair = new EntityRepairImpl( this, serializationFig );
this.entityJsonSerializer = getEntitySerializer();
+ this.columnFamily = getColumnFamily();
}
@@ -167,7 +159,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
try {
- latestEntityColumns = keyspace.prepareQuery( CF_ENTITY_DATA ).getKeySlice( rowKeys )
+ latestEntityColumns = keyspace.prepareQuery( columnFamily ).getKeySlice( rowKeys )
.withColumnRange( maxVersion, null, false, 1 ).execute().getResult()
.iterator();
}
@@ -228,7 +220,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
RowQuery<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> query =
- keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( rowKey )
+ keyspace.prepareQuery( columnFamily ).getKey( rowKey )
.withColumnRange( version, null, false, fetchSize );
return new ColumnNameIterator( query, new MvccColumnParser( entityId, entityJsonSerializer ), false );
@@ -258,7 +250,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
RowQuery<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> query =
- keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( rowKey )
+ keyspace.prepareQuery( columnFamily ).getKey( rowKey )
.withColumnRange( null, version, true, fetchSize );
return new ColumnNameIterator( query, new MvccColumnParser( entityId, entityJsonSerializer ), false );
@@ -305,7 +297,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
//create the CF entity data. We want it reversed b/c we want the most recent version at the top of the
//row for fast seeks
MultiTennantColumnFamilyDefinition cf =
- new MultiTennantColumnFamilyDefinition( CF_ENTITY_DATA, BytesType.class.getSimpleName(),
+ new MultiTennantColumnFamilyDefinition( columnFamily, BytesType.class.getSimpleName(),
ReversedType.class.getSimpleName() + "(" + UUIDType.class.getSimpleName() + ")",
BytesType.class.getSimpleName(), MultiTennantColumnFamilyDefinition.CacheOption.KEYS );
@@ -332,7 +324,7 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
ScopedRowKey.fromKey( applicationId, collectionPrefixedKey );
- op.doOp( batch.withRow( CF_ENTITY_DATA, rowKey ) );
+ op.doOp( batch.withRow( columnFamily, rowKey ) );
return batch;
}
@@ -412,4 +404,16 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
* Return the entity serializer for this instance
*/
protected abstract AbstractSerializer<EntityWrapper> getEntitySerializer();
+
+ /**
+ * Get the column family to perform operations with
+ * @return
+ */
+ protected abstract MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> getColumnFamily();
+ /**
+ *
+ private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> CF_ENTITY_DATA =
+ new MultiTennantColumnFamily<>( "Entity_Version_Data", ROW_KEY_SER, UUIDSerializer.get() );
+
+ */
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
index 49b3486..b40243d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
@@ -21,11 +21,16 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.UUID;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
@@ -39,6 +44,7 @@ import com.netflix.astyanax.model.Composites;
import com.netflix.astyanax.serializers.AbstractSerializer;
import com.netflix.astyanax.serializers.ByteBufferSerializer;
import com.netflix.astyanax.serializers.BytesArraySerializer;
+import com.netflix.astyanax.serializers.UUIDSerializer;
/**
@@ -50,6 +56,18 @@ public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializati
private static final EntitySerializer ENTITY_JSON_SER = new EntitySerializer();
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ private static final CollectionScopedRowKeySerializer<Id> ROW_KEY_SER =
+ new CollectionScopedRowKeySerializer<>( ID_SER );
+
+
+ private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> CF_ENTITY_DATA =
+ new MultiTennantColumnFamily<>( "Entity_Version_Data", ROW_KEY_SER, UUIDSerializer.get() );
+
+
+
@Inject
public MvccEntitySerializationStrategyV1Impl( final Keyspace keyspace, final SerializationFig serializationFig ) {
super( keyspace, serializationFig );
@@ -62,6 +80,12 @@ public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializati
}
+ @Override
+ protected MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> getColumnFamily() {
+ return CF_ENTITY_DATA;
+ }
+
+
public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
index f80960d..723bb38 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -21,12 +21,20 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
import java.nio.ByteBuffer;
-import java.util.Arrays;
+import java.util.UUID;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.FieldBuffer;
+import org.apache.usergrid.persistence.core.astyanax.FieldBufferBuilder;
+import org.apache.usergrid.persistence.core.astyanax.FieldBufferParser;
+import org.apache.usergrid.persistence.core.astyanax.FieldBufferSerializer;
+import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
+import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
@@ -34,12 +42,8 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.model.CompositeBuilder;
-import com.netflix.astyanax.model.CompositeParser;
-import com.netflix.astyanax.model.Composites;
import com.netflix.astyanax.serializers.AbstractSerializer;
-import com.netflix.astyanax.serializers.ByteBufferSerializer;
-import com.netflix.astyanax.serializers.BytesArraySerializer;
+import com.netflix.astyanax.serializers.UUIDSerializer;
/**
@@ -51,6 +55,20 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
private static final EntitySerializer ENTITY_JSON_SER = new EntitySerializer();
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ private static final CollectionScopedRowKeySerializer<Id> ROW_KEY_SER =
+ new CollectionScopedRowKeySerializer<>( ID_SER );
+
+
+ private static final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> CF_ENTITY_DATA =
+ new MultiTennantColumnFamily<>( "Entity_Version_Data_V2", ROW_KEY_SER, UUIDSerializer.get() );
+
+ private static final FieldBufferSerializer FIELD_BUFFER_SERIALIZER = FieldBufferSerializer.get();
+
+
+
@Inject
public MvccEntitySerializationStrategyV2Impl( final Keyspace keyspace, final SerializationFig serializationFig ) {
super( keyspace, serializationFig );
@@ -63,23 +81,29 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
}
- public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
+ @Override
+ protected MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> getColumnFamily() {
+ return CF_ENTITY_DATA;
+ }
+
+ public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
- private static final ByteBufferSerializer BUFFER_SERIALIZER = ByteBufferSerializer.get();
- private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+// private static final ByteBufferSerializer BUFFER_SERIALIZER = ByteBufferSerializer.get();
+//
+// private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
public static final SmileFactory f = new SmileFactory();
public static ObjectMapper mapper;
- private static byte[] STATE_COMPLETE = new byte[] { 0 };
- private static byte[] STATE_DELETED = new byte[] { 1 };
- private static byte[] STATE_PARTIAL = new byte[] { 2 };
+ private static byte STATE_COMPLETE = 0 ;
+ private static byte STATE_DELETED = 1 ;
+ private static byte STATE_PARTIAL = 2 ;
- private static byte[] VERSION = new byte[] { 0 };
+ private static byte VERSION = 1;
public EntitySerializer() {
@@ -101,26 +125,28 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
return null;
}
- CompositeBuilder builder = Composites.newCompositeBuilder();
+ //we always have a max of 3 fields
+ FieldBufferBuilder builder = new FieldBufferBuilder( 3 );
- builder.addBytes( VERSION );
+ builder.addByte( VERSION );
//mark this version as empty
if ( !wrapper.entity.isPresent() ) {
//we're empty
- builder.addBytes( STATE_DELETED );
+ builder.addByte( STATE_DELETED );
- return builder.build();
+
+ return FIELD_BUFFER_SERIALIZER.toByteBuffer( builder.build() );
}
//we have an entity
if ( wrapper.status == MvccEntity.Status.COMPLETE ) {
- builder.addBytes( STATE_COMPLETE );
+ builder.addByte( STATE_COMPLETE );
}
else {
- builder.addBytes( STATE_PARTIAL );
+ builder.addByte( STATE_PARTIAL );
}
try {
@@ -131,7 +157,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
throw new RuntimeException( "Unable to serialize entity", e );
}
- return builder.build();
+ return FIELD_BUFFER_SERIALIZER.toByteBuffer( builder.build() );
}
@@ -141,41 +167,44 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
/**
* We intentionally turn data corruption exceptions when we're unable to de-serialize
* the data in cassandra. If this occurs, we'll never be able to de-serialize it
- * and it should be considered lost. This is an error that is occuring due to a bug
+ * and it should be considered lost. This is an error that is occurring due to a bug
* in serializing the entity. This is a lazy recognition + repair signal for deployment with
* existing systems.
*/
- CompositeParser parser;
+
+ final FieldBuffer fieldBuffer;
+
try {
- parser = Composites.newCompositeParser( byteBuffer );
+ fieldBuffer = FIELD_BUFFER_SERIALIZER.fromByteBuffer( byteBuffer );
}
catch ( Exception e ) {
throw new DataCorruptionException( "Unable to de-serialze entity", e );
}
- byte[] version = parser.read( BYTES_ARRAY_SERIALIZER );
+ FieldBufferParser parser = new FieldBufferParser( fieldBuffer );
+
+
+
+ byte version = parser.readByte();
- if ( !Arrays.equals( VERSION, version ) ) {
+ if ( VERSION != version ) {
throw new UnsupportedOperationException( "A version of type " + version + " is unsupported" );
}
- byte[] state = parser.read( BYTES_ARRAY_SERIALIZER );
+ byte state = parser.readByte();
// it's been deleted, remove it
- if ( Arrays.equals( STATE_DELETED, state ) ) {
+ if ( STATE_DELETED == state ) {
return new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() );
}
Entity storedEntity;
- ByteBuffer jsonBytes = parser.read( BUFFER_SERIALIZER );
- byte[] array = jsonBytes.array();
- int start = jsonBytes.arrayOffset();
- int length = jsonBytes.remaining();
+ byte[] array = parser.readBytes();
try {
- storedEntity = mapper.readValue( array, start, length, Entity.class );
+ storedEntity = mapper.readValue( array, Entity.class );
}
catch ( Exception e ) {
throw new DataCorruptionException( "Unable to read entity data", e );
@@ -183,7 +212,7 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
final Optional<Entity> entity = Optional.of( storedEntity );
- if ( Arrays.equals( STATE_COMPLETE, state ) ) {
+ if ( STATE_COMPLETE == state ) {
return new EntityWrapper( MvccEntity.Status.COMPLETE, entity );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index db5e3a4..21e3260 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -731,6 +731,6 @@ public class EntityCollectionManagerIT {
final Entity loaded = manager.load( entity.getId() ).toBlocking().last();
- EntityHelper.verifySame( entity, loaded );
+ EntityHelper.verifyDeepEquals( entity, loaded );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
deleted file mode 100644
index 0d7e335..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationSTrategyV2FixTests.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.collection.serialization.impl;
-
-
-import java.util.Iterator;
-import java.util.UUID;
-
-import org.junit.Test;
-
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
-import org.apache.usergrid.persistence.collection.util.EntityHelper;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-public abstract class MvccEntitySerializationSTrategyV2FixTests extends MvccEntitySerializationStrategyImplTest {
-
-
- /**
- * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
- */
- @Test
- public void largeEntityWriteRead() throws ConnectionException {
- final int setSize = 65535 * 2;
-
- final Entity entity = EntityHelper.generateEntity( setSize );
-
- //now we have one massive, entity, save it and retrieve it.
- CollectionScope context =
- new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
-
-
- final Id simpleId = new SimpleId( "test" );
- final UUID version = UUIDGenerator.newTimeUUID();
- final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
-
- final MvccEntity mvccEntity = new MvccEntityImpl( simpleId, version, status, entity );
-
-
- getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
-
- //now load it
- final Iterator<MvccEntity> loaded =
- getMvccEntitySerializationStrategy().loadAscendingHistory( context, entity.getId(), version, 100 );
-
- assertTrue( loaded.hasNext() );
-
- final MvccEntity loadedEntity = loaded.next();
-
- assertEquals( "The loaded entity should match the stored entity", mvccEntity, loadedEntity );
-
- EntityHelper.verifySame( entity, loadedEntity.getEntity().get() );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index d54d958..0cdc320 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -35,9 +35,11 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
@@ -579,6 +581,58 @@ public abstract class MvccEntitySerializationStrategyImplTest {
}
+
+ /**
+ * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
+ */
+ @Test
+ public void largeEntityWriteRead() throws ConnectionException {
+ final int setSize = 65535 * 2;
+
+ final Entity entity = EntityHelper.generateEntity( setSize );
+
+ //now we have one massive, entity, save it and retrieve it.
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "parent" ), "tests" );
+
+
+ final Id id = entity.getId();
+ ValidationUtils.verifyIdentity( id );
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
+
+ final MvccEntity mvccEntity = new MvccEntityImpl( id, version, status, entity );
+
+
+ getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
+
+ //now load it
+ final Iterator<MvccEntity> loaded =
+ getMvccEntitySerializationStrategy().loadDescendingHistory( context, id, version, 100 );
+
+
+ assertLargeEntity( mvccEntity, loaded );
+
+ MvccEntity returned = serializationStrategy.load( context, Collections.singleton( id ) , version ).getEntity( id );
+
+ assertLargeEntity( mvccEntity, returned );
+ }
+
+
+ /**
+ * Assertion for the loaded values of a large entity. Note the returned may be nullable
+ * @param expected
+ * @param returned
+ */
+ protected abstract void assertLargeEntity(final MvccEntity expected, final Iterator<MvccEntity> returned);
+
+ /**
+ * Tests large entity that is directly returned
+ * @param expected
+ * @param returned
+ */
+ protected abstract void assertLargeEntity(final MvccEntity expected, final MvccEntity returned);
+
@Test(expected = NullPointerException.class)
public void writeParamsContext() throws ConnectionException {
serializationStrategy.write( null, mock( MvccEntity.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
index 331bea2..df5f7d1 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV1Test.java
@@ -20,10 +20,13 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.Iterator;
+
import org.junit.After;
import org.junit.Before;
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.core.guice.PreviousImpl;
@@ -34,6 +37,9 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -69,6 +75,25 @@ public class MvccEntitySerializationStrategyProxyV1Test extends MvccEntitySerial
}
+
+ @Override
+ protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
+ //known bug in v1, it should return null. Fixed in v2
+ assertTrue( "V1 should be broken, but still have a next", returned.hasNext() );
+
+ final MvccEntity entity = returned.next();
+
+
+ assertLargeEntity( expected, entity );
+ }
+
+
+ protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+ assertEquals( "Marked as deleted since we can't parse", MvccEntity.Status.DELETED, returned.getStatus() );
+
+ assertFalse( "V1 returns empty b/c parse fails", returned.getEntity().isPresent() );
+ }
+
@After
public void reSetMigrationVersion() {
migrationInfoSerialization.setVersion( existingVersion );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
index 02eccb1..dc31ff7 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyV2Test.java
@@ -20,12 +20,16 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.Iterator;
+
import org.junit.After;
import org.junit.Before;
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
import org.apache.usergrid.persistence.core.test.ITRunner;
@@ -33,10 +37,12 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
+import static org.junit.Assert.assertTrue;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerializationSTrategyV2FixTests {
+public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerializationStrategyImplTest {
@Inject
@ProxyImpl
@@ -68,6 +74,25 @@ public class MvccEntitySerializationStrategyProxyV2Test extends MvccEntitySerial
}
+ @Override
+ protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
+ assertTrue( returned.hasNext() );
+
+ final MvccEntity loadedEntity = returned.next();
+
+ assertLargeEntity( expected, loadedEntity );
+ }
+
+
+ @Override
+ protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+
+ org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
+
+ EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
+ }
+
+
@After
public void reSetMigrationVersion() {
migrationInfoSerialization.setVersion( existingVersion );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
index 01d9a9b..9b0ccfb 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1ImplTest.java
@@ -20,8 +20,11 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.Iterator;
+
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.core.guice.PreviousImpl;
@@ -30,6 +33,10 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
@@ -39,6 +46,26 @@ public class MvccEntitySerializationStrategyV1ImplTest extends MvccEntitySeriali
@PreviousImpl
private MvccEntitySerializationStrategy serializationStrategy;
+
+ @Override
+ protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
+ //known bug in v1, it should return null. Fixed in v2
+ assertTrue( "V1 should be broken, but still have a next", returned.hasNext() );
+
+ final MvccEntity entity = returned.next();
+
+
+ assertLargeEntity( expected, entity );
+ }
+
+
+ protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+ assertEquals( "Marked as deleted since we can't parse", MvccEntity.Status.DELETED, returned.getStatus() );
+
+ assertFalse( "V1 returns empty b/c parse fails", returned.getEntity().isPresent() );
+ }
+
+
@Override
protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
return serializationStrategy;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
index 012eeff..53952d0 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2ImplTest.java
@@ -20,30 +20,53 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.Iterator;
+
import org.junit.runner.RunWith;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.util.EntityHelper;
import org.apache.usergrid.persistence.core.guice.CurrentImpl;
-import org.apache.usergrid.persistence.core.guice.PreviousImpl;
import org.apache.usergrid.persistence.core.test.ITRunner;
import org.apache.usergrid.persistence.core.test.UseModules;
import com.google.inject.Inject;
+import static org.junit.Assert.assertTrue;
+
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
-public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySerializationSTrategyV2FixTests {
+public class MvccEntitySerializationStrategyV2ImplTest extends MvccEntitySerializationStrategyImplTest {
@Inject
@CurrentImpl
private MvccEntitySerializationStrategy serializationStrategy;
+
@Override
protected MvccEntitySerializationStrategy getMvccEntitySerializationStrategy() {
return serializationStrategy;
}
+ @Override
+ protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
+ assertTrue( returned.hasNext() );
+
+ final MvccEntity loadedEntity = returned.next();
+
+ assertLargeEntity( expected, loadedEntity );
+ }
+
+
+ @Override
+ protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+
+ org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
+
+ EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
index 4eb09ab..5c99121 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/EntityHelper.java
@@ -81,7 +81,7 @@ public class EntityHelper {
* @param expected
* @param returned
*/
- public static void verifySame(final Entity expected, final Entity returned){
+ public static void verifyDeepEquals( final Entity expected, final Entity returned ){
//perform object equals
assertEquals("Expected same entity equality", expected, returned);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
index ef3b962..68bb32b 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
@@ -50,7 +50,7 @@ public class FieldBufferParser {
/**
* Return the value as an integer
*/
- public int getInteger() {
+ public int readInteger() {
return INTEGER_SERIALIZER.fromByteBuffer( getNext() );
}
@@ -58,7 +58,7 @@ public class FieldBufferParser {
/**
* Return the value as a byte array
*/
- public byte[] getBytes() {
+ public byte[] readBytes() {
return BYTES_ARRAY_SERIALIZER.fromByteBuffer( getNext() );
}
@@ -66,7 +66,7 @@ public class FieldBufferParser {
/**
* return the next vlaue as a byte
*/
- public byte getByte() {
+ public byte readByte() {
return BYTE_SERIALIZER.fromByteBuffer( getNext() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68f4e0f1/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
index e9a8194..850bcdd 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
@@ -60,15 +60,15 @@ public class FieldBufferSerializerTest {
FieldBufferParser parser = new FieldBufferParser( parsed );
- final byte returnedByte = parser.getByte();
+ final byte returnedByte = parser.readByte();
assertEquals( "Bytes should be equal", setByte, returnedByte );
- final int returnedInt = parser.getInteger();
+ final int returnedInt = parser.readInteger();
assertEquals( "Integer should be equal", setInteger, returnedInt );
- final byte[] returnedByteArray = parser.getBytes();
+ final byte[] returnedByteArray = parser.readBytes();
assertArrayEquals( "arrays should be equal", setByteArray, returnedByteArray );
}
@@ -101,15 +101,15 @@ public class FieldBufferSerializerTest {
FieldBufferParser parser = new FieldBufferParser( parsed );
- final byte[] returnedArray = parser.getBytes();
+ final byte[] returnedArray = parser.readBytes();
assertArrayEquals( setByteArray, returnedArray );
- final byte returnedByte = parser.getByte();
+ final byte returnedByte = parser.readByte();
assertEquals( "Bytes should be equal", setByte, returnedByte );
- final int returnedInt = parser.getInteger();
+ final int returnedInt = parser.readInteger();
assertEquals( "Integer should be equal", setInteger, returnedInt );
}
[13/19] incubator-usergrid git commit: Refactored generation of
collection scope to be re-usable with CpNamingUtils for consistency across
modules
Posted by sf...@apache.org.
Refactored generation of collection scope to be re-usable with CpNamingUtils for consistency across modules
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/364a6ea6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/364a6ea6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/364a6ea6
Branch: refs/heads/two-dot-o
Commit: 364a6ea6737194a1dd266b6d0e89abb4a40ddf96
Parents: 9ed7964
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 20 14:24:03 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 20 14:24:03 2014 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 44 ++++++--------------
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../corepersistence/CpRelationManager.java | 35 ++++------------
.../corepersistence/util/CpNamingUtils.java | 19 +++++++++
4 files changed, 41 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/364a6ea6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index e2f67e8..e6b8bce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -131,6 +131,7 @@ import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
import static org.apache.usergrid.persistence.Schema.COLLECTION_USERS;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_PERMISSIONS;
@@ -372,13 +373,9 @@ public class CpEntityManager implements EntityManager {
}
Id id = new SimpleId( entityRef.getUuid(), entityRef.getType() );
- String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() );
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- collectionName );
+ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), entityRef.getType());
- EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
// if ( !UUIDUtils.isTimeBased( id.getUuid() ) ) {
// throw new IllegalArgumentException(
@@ -457,13 +454,10 @@ public class CpEntityManager implements EntityManager {
String type = Schema.getDefaultSchema().getEntityType( entityClass );
Id id = new SimpleId( entityId, type );
- String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( type );
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- collectionName );
- EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
+ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), type);
+
// if ( !UUIDUtils.isTimeBased( id.getUuid() ) ) {
// throw new IllegalArgumentException(
@@ -522,9 +516,8 @@ public class CpEntityManager implements EntityManager {
public void update( Entity entity ) throws Exception {
// first, update entity index in its own collection scope
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entity.getType() ) );
+
+ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), entity.getType());
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
@@ -590,9 +583,7 @@ public class CpEntityManager implements EntityManager {
private Observable deleteAsync( EntityRef entityRef ) throws Exception {
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+ CollectionScope collectionScope =getCollectionScopeNameFromEntityType(applicationScope.getApplication(), entityRef.getType() );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -639,7 +630,7 @@ public class CpEntityManager implements EntityManager {
// deindex from default index scope
IndexScope defaultIndexScope = new IndexScopeImpl( getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+ getCollectionScopeNameFromEntityType( entityRef.getType() ) );
batch.deindex( defaultIndexScope, entity );
@@ -1010,15 +1001,10 @@ public class CpEntityManager implements EntityManager {
@Override
public void deleteProperty( EntityRef entityRef, String propertyName ) throws Exception {
-
- String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() );
-
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- collectionName );
+ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(getApplicationScope().getApplication(), entityRef.getType());
IndexScope defaultIndexScope = new IndexScopeImpl( getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+ getCollectionScopeNameFromEntityType( entityRef.getType() ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() );
@@ -2187,10 +2173,7 @@ public class CpEntityManager implements EntityManager {
private Id getIdForUniqueEntityField( final String collectionName, final String propertyName,
final Object propertyValue ) {
- CollectionScope collectionScope =
- new CollectionScopeImpl( applicationScope.getApplication(), applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( collectionName ) );
-
+ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), collectionName);
final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@ -2514,9 +2497,8 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity cpEntity = entityToCpEntity( entity, importId );
// prepare to write and index Core Persistence Entity into default scope
- CollectionScope collectionScope =
- new CollectionScopeImpl( applicationScope.getApplication(), applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( eType ) );
+ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), eType);
+
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
if ( logger.isDebugEnabled() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/364a6ea6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e07bb00..82e80a5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -541,7 +541,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public long performEntityCount() {
//TODO, this really needs to be a task that writes this data somewhere since this will get
//progressively slower as the system expands
- return AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache ).longCount().toBlocking().last();
+ return AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 ).longCount().toBlocking().last();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/364a6ea6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 8514504..dcb4ba1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -120,6 +120,7 @@ import rx.functions.Func1;
import static java.util.Arrays.asList;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_TYPES;
@@ -219,10 +220,7 @@ public class CpRelationManager implements RelationManager {
this.indexBucketLocator = indexBucketLocator; // TODO: this also
// load the Core Persistence version of the head entity as well
- this.headEntityScope = new CollectionScopeImpl(
- this.applicationScope.getApplication(),
- this.applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( headEntity.getType() ) );
+ this.headEntityScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), headEntity.getType());
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading head entity {}:{} from scope\n app {}\n owner {}\n name {}",
@@ -606,10 +604,7 @@ public class CpRelationManager implements RelationManager {
public Entity addToCollection( String collName, EntityRef itemRef, boolean connectBack )
throws Exception {
- CollectionScope memberScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ) );
+ CollectionScope memberScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), itemRef.getType());
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
org.apache.usergrid.persistence.model.entity.Entity memberEntity =
@@ -642,10 +637,7 @@ public class CpRelationManager implements RelationManager {
}
// load the new member entity to be added to the collection from its default scope
- CollectionScope memberScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ) );
+ CollectionScope memberScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), itemRef.getType());
//TODO, this double load should disappear once events are in
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
@@ -797,11 +789,7 @@ public class CpRelationManager implements RelationManager {
}
// load the entity to be removed to the collection
- CollectionScope memberScope = new CollectionScopeImpl(
- this.applicationScope.getApplication(),
- this.applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ) );
- EntityCollectionManager memberMgr = managerCache.getEntityCollectionManager( memberScope );
+ CollectionScope memberScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), itemRef.getType());
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading entity to remove from collection "
@@ -1017,10 +1005,7 @@ public class CpRelationManager implements RelationManager {
ConnectionRefImpl connection = new ConnectionRefImpl( headEntity, connectionType, connectedEntityRef );
- CollectionScope targetScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( connectedEntityRef.getType() ) );
+ CollectionScope targetScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), connectedEntityRef.getType());
if ( logger.isDebugEnabled() ) {
logger.debug("createConnection(): "
@@ -1249,12 +1234,8 @@ public class CpRelationManager implements RelationManager {
String connectionType = connectionRef.getConnectedEntity().getConnectionType();
- CollectionScope targetScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( connectedEntityRef.getType()) );
-
- EntityCollectionManager targetEcm = managerCache.getEntityCollectionManager( targetScope );
+ CollectionScope targetScope = getCollectionScopeNameFromEntityType( applicationScope.getApplication(),
+ connectedEntityRef.getType() );
if ( logger.isDebugEnabled() ) {
logger.debug( "Deleting connection '{}' from source {}:{} \n to target {}:{}",
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/364a6ea6/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 6fb35a3..684b6e0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.corepersistence.util;
import java.util.UUID;
import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
@@ -70,6 +72,23 @@ public class CpNamingUtils {
public static String TYPES_BY_UUID_MAP = "zzz_typesbyuuid_zzz";
+ /**
+ * Generate a collection scope for a collection within the application's Id for the given type
+ * @param applicationId The applicationId that owns this entity
+ * @param type The type in the collection
+ * @return The collectionScope
+ */
+ public static CollectionScope getCollectionScopeNameFromEntityType(final Id applicationId, final String type){
+ return
+ new CollectionScopeImpl( applicationId, applicationId,
+ getCollectionScopeNameFromEntityType( type ) );
+ }
+
+ /**
+ * Get the collection name from the entity/id type
+ * @param type
+ * @return
+ */
public static String getCollectionScopeNameFromEntityType( String type ) {
String csn = EDGE_COLL_SUFFIX + Schema.defaultCollectionName( type );
return csn.toLowerCase();
[14/19] incubator-usergrid git commit: Finished migration testing.
All core tests pass.
Posted by sf...@apache.org.
Finished migration testing. All core tests pass.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e32d5212
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e32d5212
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e32d5212
Branch: refs/heads/two-dot-o
Commit: e32d5212af3c53cc84c9a505cb9bb6824e6a16f2
Parents: 364a6ea
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 20 15:48:19 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 20 15:48:19 2014 -0700
----------------------------------------------------------------------
.../migration/EntityDataMigration.java | 90 ++++---
.../rx/ApplicationObservable.java | 65 +++--
.../migration/EntityDataMigrationIT.java | 251 +++++++++++++++++++
.../migration/GraphShardVersionMigrationIT.java | 131 +++++-----
4 files changed, 394 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e32d5212/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java
index 79d31a8..767574d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/EntityDataMigration.java
@@ -22,9 +22,7 @@ package org.apache.usergrid.corepersistence.migration;
import java.util.Iterator;
import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable;
@@ -53,9 +51,6 @@ import rx.functions.Action1;
public class EntityDataMigration implements DataMigration {
- private static final Logger logger = LoggerFactory.getLogger( EntityDataMigration.class );
-
-
private final MvccEntitySerializationStrategy v1Serialization;
private final MvccEntitySerializationStrategy v2Serialization;
@@ -77,58 +72,71 @@ public class EntityDataMigration implements DataMigration {
@Override
public void migrate( final ProgressObserver observer ) throws Throwable {
+ final AtomicLong atomicLong = new AtomicLong();
+
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 )
+ .doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
- AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 ).doOnNext(
- new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
+ @Override
+ public void call(
+ final AllEntitiesInSystemObservable.ApplicationEntityGroup
+ applicationEntityGroup ) {
- @Override
- public void call(
- final AllEntitiesInSystemObservable.ApplicationEntityGroup applicationEntityGroup ) {
+ final UUID now = UUIDGenerator.newTimeUUID();
- final UUID now = UUIDGenerator.newTimeUUID();
+ final Id appScopeId =
+ applicationEntityGroup.applicationScope.getApplication();
- final Id appScopeId = applicationEntityGroup.applicationScope.getApplication();
+ final MutationBatch totalBatch = keyspace.prepareMutationBatch();
- final MutationBatch totalBatch = keyspace.prepareMutationBatch();
+ //go through each entity in the system, and load it's entire
+ // history
+ for ( Id entityId : applicationEntityGroup.entityIds ) {
- for ( Id entityId : applicationEntityGroup.entityIds ) {
+ CollectionScope currentScope = CpNamingUtils
+ .getCollectionScopeNameFromEntityType( appScopeId,
+ entityId.getType() );
- CollectionScope currentScope = CpNamingUtils.getCollectionScopeNameFromEntityType(
- appScopeId, entityId.getType() );
+ //for each element in the history in the previous version,
+ // copy it to the CF in v2
+ Iterator<MvccEntity> allVersions = v1Serialization
+ .loadDescendingHistory( currentScope, entityId, now,
+ 1000 );
- Iterator<MvccEntity> allVersions =
- v1Serialization.loadDescendingHistory( currentScope, entityId, now, 1000 );
+ while ( allVersions.hasNext() ) {
+ final MvccEntity version = allVersions.next();
- while ( allVersions.hasNext() ) {
- final MvccEntity version = allVersions.next();
+ final MutationBatch versionBatch =
+ v2Serialization.write( currentScope, version );
- final MutationBatch versionBatch = v2Serialization.write( currentScope, version );
+ totalBatch.mergeShallow( versionBatch );
+
+ if ( atomicLong.incrementAndGet() % 50 == 0 ) {
+ executeBatch( totalBatch, observer, atomicLong );
+ }
+ }
+ }
+
+ executeBatch( totalBatch, observer, atomicLong );
+ }
+ } ).toBlocking().last();
+ }
- totalBatch.mergeShallow( versionBatch );
- if ( totalBatch.getRowCount() >= 50 ) {
- try {
- totalBatch.execute();
- }
- catch ( ConnectionException e ) {
- throw new DataMigrationException( "Unable to migrate batches ", e );
- }
- }
- }
- }
+ private void executeBatch( final MutationBatch batch, final ProgressObserver po, final AtomicLong count ) {
+ try {
+ batch.execute();
- try {
- totalBatch.execute();
- }
- catch ( ConnectionException e ) {
- throw new DataMigrationException( "Unable to migrate batches ", e );
- }
- }
- } ).toBlocking().last();
+ po.update( getVersion(), "Finished copying " + count + " entities to the new format" );
+ }
+ catch ( ConnectionException e ) {
+ po.failed( getVersion(), "Failed to execute mutation in cassandra" );
+ throw new DataMigrationException( "Unable to migrate batches ", e );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e32d5212/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
index 898812b..6019bca 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
@@ -23,18 +23,20 @@ package org.apache.usergrid.corepersistence.rx;
import java.util.Arrays;
import java.util.UUID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
@@ -47,14 +49,12 @@ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicat
/**
* An observable that will emit all application stored in the system.
*/
-public class ApplicationObservable {
-
+public class ApplicationObservable {
+ private static final Logger logger = LoggerFactory.getLogger( ApplicationObservable.class );
/**
* Get all applicationIds as an observable
- * @param managerCache
- * @return
*/
public static Observable<Id> getAllApplicationIds( final ManagerCache managerCache ) {
@@ -62,22 +62,20 @@ public class ApplicationObservable {
//this way consumers can perform whatever work they need to on the root system first
- final Observable<Id> systemIds = Observable.from( Arrays.asList( generateApplicationId( CpNamingUtils.DEFAULT_APPLICATION_ID ),
- generateApplicationId( CpNamingUtils.MANAGEMENT_APPLICATION_ID ),
- generateApplicationId( CpNamingUtils.SYSTEM_APP_ID ) ) );
-
-
+ final Observable<Id> systemIds = Observable.from( Arrays
+ .asList( generateApplicationId( CpNamingUtils.DEFAULT_APPLICATION_ID ),
+ generateApplicationId( CpNamingUtils.MANAGEMENT_APPLICATION_ID ),
+ generateApplicationId( CpNamingUtils.SYSTEM_APP_ID ) ) );
final ApplicationScope appScope = getApplicationScope( CpNamingUtils.SYSTEM_APP_ID );
- final CollectionScope appInfoCollectionScope = new CollectionScopeImpl(
- appScope.getApplication(),
- appScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPINFOS ));
+ final CollectionScope appInfoCollectionScope =
+ new CollectionScopeImpl( appScope.getApplication(), appScope.getApplication(),
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( CpNamingUtils.APPINFOS ) );
- final EntityCollectionManager
- collectionManager = managerCache.getEntityCollectionManager( appInfoCollectionScope );
+ final EntityCollectionManager collectionManager =
+ managerCache.getEntityCollectionManager( appInfoCollectionScope );
final GraphManager gm = managerCache.getGraphManager( appScope );
@@ -97,21 +95,34 @@ public class ApplicationObservable {
//get the app info and load it
final Id appInfo = edge.getTargetNode();
- return collectionManager.load( appInfo ).map( new Func1<org.apache.usergrid.persistence.model.entity.Entity, Id>() {
+ return collectionManager.load( appInfo )
+ //filter out null entities
+ .filter( new Func1<Entity, Boolean>() {
+ @Override
+ public Boolean call( final Entity entity ) {
+ if ( entity == null ) {
+ logger.warn( "Encountered a null application info for id {}", appInfo );
+ return false;
+ }
+ return true;
+ }
+ } )
+ //get the id from the entity
+ .map( new Func1<org.apache.usergrid.persistence.model.entity.Entity, Id>() {
- @Override
- public Id call( final org.apache.usergrid.persistence.model.entity.Entity entity ) {
- final UUID uuid = (UUID )entity.getField( "applicationUuid" ).getValue();
- return CpNamingUtils.generateApplicationId(uuid);
- }
- } );
+ @Override
+ public Id call( final org.apache.usergrid.persistence.model.entity.Entity entity ) {
+
+ final UUID uuid = ( UUID ) entity.getField( "applicationUuid" ).getValue();
+
+ return CpNamingUtils.generateApplicationId( uuid );
+ }
+ } );
}
} );
- return Observable.merge( systemIds, appIds);
+ return Observable.merge( systemIds, appIds );
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e32d5212/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityDataMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityDataMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityDataMigrationIT.java
new file mode 100644
index 0000000..5c9e14c
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/EntityDataMigrationIT.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.migration;
+
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.corepersistence.EntityWriteHelper;
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.corepersistence.rx.AllEntitiesInSystemObservable;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.guice.PreviousImpl;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManagerImpl;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Injector;
+import com.google.inject.Key;
+
+import rx.functions.Action1;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+public class EntityDataMigrationIT extends AbstractCoreIT {
+
+
+ private Injector injector;
+
+
+ private EntityDataMigration entityDataMigration;
+ private ManagerCache managerCache;
+ private DataMigrationManager dataMigrationManager;
+ private MigrationInfoSerialization migrationInfoSerialization;
+ private MvccEntitySerializationStrategy v1Strategy;
+ private MvccEntitySerializationStrategy v2Strategy;
+ private EntityManagerFactory emf;
+
+
+ @Before
+ public void setup() {
+ emf = setup.getEmf();
+ injector = CpSetup.getInjector();
+ entityDataMigration = injector.getInstance( EntityDataMigration.class );
+ managerCache = injector.getInstance( ManagerCache.class );
+ dataMigrationManager = injector.getInstance( DataMigrationManager.class );
+ migrationInfoSerialization = injector.getInstance( MigrationInfoSerialization.class );
+ v1Strategy = injector.getInstance( Key.get(MvccEntitySerializationStrategy.class, PreviousImpl.class) );
+ v2Strategy = injector.getInstance( Key.get(MvccEntitySerializationStrategy.class, CurrentImpl.class) );
+ }
+
+
+ @Test
+ public void testDataMigration() throws Throwable {
+
+ assertEquals( "version 3 expected", 3, entityDataMigration.getVersion() );
+
+
+ /**
+ * Reset to our version -1 and start the migration
+ */
+ dataMigrationManager.resetToVersion( entityDataMigration.getVersion() - 1 );
+
+
+ final EntityManager newAppEm = app.getEntityManager();
+
+ final String type1 = "type1thing";
+ final String type2 = "type2thing";
+ final int size = 10;
+
+ final Set<Id> type1Identities = EntityWriteHelper.createTypes( newAppEm, type1, size );
+ final Set<Id> type2Identities = EntityWriteHelper.createTypes( newAppEm, type2, size );
+
+
+ final Set<Id> createdEntityIds = new HashSet<>();
+ createdEntityIds.addAll( type1Identities );
+ createdEntityIds.addAll( type2Identities );
+
+
+ final TestProgressObserver progressObserver = new TestProgressObserver();
+
+
+ //load everything that appears in v1, migrate and ensure it appears in v2
+ final Set<MvccEntity> savedEntities = new HashSet<>( 10000 );
+ //set that holds all entityIds for later assertion
+ final Set<Id> entityIds = new HashSet<>(10000);
+
+
+ //read everything in previous version format and put it into our types. Assumes we're
+ //using a test system, and it's not a huge amount of data, otherwise we'll overflow.
+
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 )
+ .doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
+ @Override
+ public void call(
+ final AllEntitiesInSystemObservable.ApplicationEntityGroup entity ) {
+
+ //add all versions from history to our comparison
+ for ( final Id id : entity.entityIds ) {
+
+ CollectionScope scope = CpNamingUtils
+ .getCollectionScopeNameFromEntityType(
+ entity.applicationScope.getApplication(),
+ id.getType() );
+
+ final Iterator<MvccEntity> versions = v1Strategy
+ .loadDescendingHistory( scope, id, UUIDGenerator.newTimeUUID(),
+ 100 );
+
+ while ( versions.hasNext() ) {
+
+ final MvccEntity mvccEntity = versions.next();
+
+ savedEntities.add( mvccEntity );
+
+ createdEntityIds.remove( mvccEntity.getId() );
+
+ entityIds.add( id );
+ }
+ }
+ }
+ } ).toBlocking().lastOrDefault( null );
+
+ assertEquals( "Newly saved entities encountered", 0, createdEntityIds.size() );
+ assertTrue( "Saved new entities", savedEntities.size() > 0 );
+
+ //perform the migration
+ entityDataMigration.migrate( progressObserver );
+
+ assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
+ assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
+
+
+ //write the status and version, then invalidate the cache so it appears
+ migrationInfoSerialization.setStatusCode( DataMigrationManagerImpl.StatusCode.COMPLETE.status );
+ migrationInfoSerialization.setVersion( entityDataMigration.getVersion() );
+ dataMigrationManager.invalidate();
+
+ assertEquals( "New version saved, and we should get new implementation", entityDataMigration.getVersion(),
+ dataMigrationManager.getCurrentVersion() );
+
+
+ //now visit all entities in the system again, load them from v2, and ensure they're the same
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 )
+ .doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
+ @Override
+ public void call(
+ final AllEntitiesInSystemObservable
+ .ApplicationEntityGroup entity ) {
+ //add all versions from history to our comparison
+ for ( final Id id : entity.entityIds ) {
+
+ CollectionScope scope = CpNamingUtils
+ .getCollectionScopeNameFromEntityType(
+ entity.applicationScope.getApplication(),
+ id.getType() );
+
+ final Iterator<MvccEntity> versions = v2Strategy
+ .loadDescendingHistory( scope, id,
+ UUIDGenerator.newTimeUUID(), 100 );
+
+ while ( versions.hasNext() ) {
+
+ final MvccEntity mvccEntity = versions.next();
+
+ savedEntities.remove( mvccEntity );
+ }
+ }
+ }
+ }
+
+
+ ).toBlocking().lastOrDefault( null );
+
+
+ assertEquals( "All entities migrated", 0, savedEntities.size() );
+
+
+ //now visit all entities in the system again, and load them from the EM, ensure we see everything we did in the v1 traversal
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 )
+ .doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
+ @Override
+ public void call(
+ final AllEntitiesInSystemObservable
+ .ApplicationEntityGroup entity ) {
+
+ final EntityManager em = emf.getEntityManager( entity.applicationScope.getApplication().getUuid() );
+
+ //add all versions from history to our comparison
+ for ( final Id id : entity.entityIds ) {
+
+
+ try {
+ final Entity emEntity = em.get( SimpleEntityRef.fromId( id ) );
+
+ if(emEntity != null){
+ entityIds.remove( id );
+ }
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException("Error loading entity", e);
+ }
+ }
+ }
+ }
+
+
+ ).toBlocking().lastOrDefault( null );
+
+
+ assertEquals("All entities could be loaded by the entity manager", 0, entityIds.size());
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e32d5212/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
index 88c02cd..51ea052 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
@@ -42,7 +42,6 @@ import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.google.inject.Injector;
-import com.netflix.astyanax.Keyspace;
import rx.functions.Action1;
@@ -55,7 +54,6 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
private Injector injector;
private GraphShardVersionMigration graphShardVersionMigration;
- private Keyspace keyspace;
private ManagerCache managerCache;
private DataMigrationManager dataMigrationManager;
private MigrationInfoSerialization migrationInfoSerialization;
@@ -65,7 +63,6 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
public void setup() {
injector = CpSetup.getInjector();
graphShardVersionMigration = injector.getInstance( GraphShardVersionMigration.class );
- keyspace = injector.getInstance( Keyspace.class );
managerCache = injector.getInstance( ManagerCache.class );
dataMigrationManager = injector.getInstance( DataMigrationManager.class );
migrationInfoSerialization = injector.getInstance( MigrationInfoSerialization.class );
@@ -75,15 +72,12 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
@Test
public void testIdMapping() throws Throwable {
- assertEquals("version 2 expected", 2, graphShardVersionMigration.getVersion());
+ assertEquals( "version 2 expected", 2, graphShardVersionMigration.getVersion() );
/**
* Reset to our version -1 and start the migration
*/
- dataMigrationManager.resetToVersion( graphShardVersionMigration.getVersion()-1 );
-
-
-
+ dataMigrationManager.resetToVersion( graphShardVersionMigration.getVersion() - 1 );
final EntityManager newAppEm = app.getEntityManager();
@@ -111,15 +105,16 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
//read everything in previous version format and put it into our types.
- AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000)
+ AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 )
.doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
@Override
- public void call( final AllEntitiesInSystemObservable.ApplicationEntityGroup entity ) {
+ public void call(
+ final AllEntitiesInSystemObservable.ApplicationEntityGroup entity ) {
final GraphManager gm =
managerCache.getGraphManager( entity.applicationScope );
- for(final Id id: entity.entityIds) {
+ for ( final Id id : entity.entityIds ) {
/**
* Get our edge types from the source
*/
@@ -135,8 +130,7 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
/**
* Get the edge types to the target
*/
- gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( id,
- null, null ) )
+ gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( id, null, null ) )
.doOnNext( new Action1<String>() {
@Override
public void call( final String s ) {
@@ -153,7 +147,7 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
//perform the migration
graphShardVersionMigration.migrate( progressObserver );
- assertEquals("Newly saved entities encounterd", 0, allEntities.size());
+ assertEquals( "Newly saved entities encounterd", 0, allEntities.size() );
assertFalse( "Progress observer should not have failed", progressObserver.getFailed() );
assertTrue( "Progress observer should have update messages", progressObserver.getUpdates().size() > 0 );
@@ -163,70 +157,57 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
migrationInfoSerialization.setVersion( graphShardVersionMigration.getVersion() );
dataMigrationManager.invalidate();
- assertEquals("New version saved, and we should get new implementation", graphShardVersionMigration.getVersion(), dataMigrationManager.getCurrentVersion());
+ assertEquals( "New version saved, and we should get new implementation",
+ graphShardVersionMigration.getVersion(), dataMigrationManager.getCurrentVersion() );
//now visit all nodes in the system and remove their types from the multi maps, it should be empty at the end
AllEntitiesInSystemObservable.getAllEntitiesInSystem( managerCache, 1000 )
.doOnNext( new Action1<AllEntitiesInSystemObservable.ApplicationEntityGroup>() {
- @Override
- public void call(
- final AllEntitiesInSystemObservable.ApplicationEntityGroup entity ) {
-
- final GraphManager gm =
- managerCache.getGraphManager( entity.applicationScope );
-
- for ( final Id id : entity.entityIds ) {
- /**
- * Get our edge types from the source
- */
- gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( id, null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- sourceTypes.remove( id, s );
- }
- } ).toBlocking().lastOrDefault( null );
-
-
- /**
- * Get the edge types to the target
- */
- gm.getEdgeTypesToTarget(
- new SimpleSearchEdgeType( id, null, null ) )
- .doOnNext( new Action1<String>() {
- @Override
- public void call( final String s ) {
- targetTypes.remove( id, s );
- }
- } ).toBlocking().lastOrDefault( null );
- }
- }
- }
-
-
- ).
-
-
- toBlocking()
-
-
- .
-
-
- lastOrDefault( null );
-
-
- assertEquals( "All source types migrated",0,sourceTypes.size( )
-
-
- );
-
-
- assertEquals( "All target types migrated",0,targetTypes.size( )
-
-
- );
- }
+ @Override
+ public void call(
+ final AllEntitiesInSystemObservable
+ .ApplicationEntityGroup entity ) {
+
+ final GraphManager gm =
+ managerCache.getGraphManager( entity.applicationScope );
+
+ for ( final Id id : entity.entityIds ) {
+ /**
+ * Get our edge types from the source
+ */
+ gm.getEdgeTypesFromSource(
+ new SimpleSearchEdgeType( id, null, null ) )
+ .doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String s ) {
+ sourceTypes.remove( id, s );
+ }
+ } ).toBlocking().lastOrDefault( null );
+
+
+ /**
+ * Get the edge types to the target
+ */
+ gm.getEdgeTypesToTarget(
+ new SimpleSearchEdgeType( id, null, null ) )
+ .doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String s ) {
+ targetTypes.remove( id, s );
+ }
+ } ).toBlocking().lastOrDefault( null );
+ }
+ }
+ }
+
+
+ ).toBlocking().lastOrDefault( null );
+
+
+ assertEquals( "All source types migrated", 0, sourceTypes.size() );
+
+
+ assertEquals( "All target types migrated", 0, targetTypes.size() );
}
+}
[17/19] incubator-usergrid git commit: Added tests for when we have
to make multiple trips to Cassandra.
Posted by sf...@apache.org.
Added tests for when we have to make multiple trips to Cassandra.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/56ce7ce6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/56ce7ce6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/56ce7ce6
Branch: refs/heads/two-dot-o
Commit: 56ce7ce6df21b08282aa484ead6c3a19397f0d14
Parents: 3e625b1
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 21 13:32:52 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 21 13:32:52 2014 -0700
----------------------------------------------------------------------
.../MvccEntitySerializationStrategyImpl.java | 132 ++++++++++++++-----
.../MvccEntitySerializationStrategyV1Impl.java | 6 +-
.../MvccEntitySerializationStrategyV2Impl.java | 5 +-
.../MvccEntitySerializationStrategyV2Test.java | 101 +++++++++++---
4 files changed, 188 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56ce7ce6/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index f3f4c13..c9ec9a8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -42,9 +42,9 @@ import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImp
import org.apache.usergrid.persistence.collection.serialization.EntityRepair;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator;
import org.apache.usergrid.persistence.core.astyanax.ColumnParser;
-import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
@@ -61,9 +61,15 @@ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.ColumnList;
import com.netflix.astyanax.model.Row;
+import com.netflix.astyanax.model.Rows;
import com.netflix.astyanax.query.RowQuery;
import com.netflix.astyanax.serializers.AbstractSerializer;
-import com.netflix.astyanax.serializers.UUIDSerializer;
+
+import rx.Observable;
+import rx.Scheduler;
+import rx.functions.Func1;
+import rx.functions.Func2;
+import rx.schedulers.Schedulers;
/**
@@ -76,14 +82,17 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
protected final Keyspace keyspace;
protected final SerializationFig serializationFig;
+ protected final CassandraFig cassandraFig;
protected final EntityRepair repair;
private final MultiTennantColumnFamily<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> columnFamily;
@Inject
- public MvccEntitySerializationStrategyImpl( final Keyspace keyspace, final SerializationFig serializationFig ) {
+ public MvccEntitySerializationStrategyImpl( final Keyspace keyspace, final SerializationFig serializationFig,
+ final CassandraFig cassandraFig ) {
this.keyspace = keyspace;
this.serializationFig = serializationFig;
+ this.cassandraFig = cassandraFig;
this.repair = new EntityRepairImpl( this, serializationFig );
this.columnFamily = getColumnFamily();
}
@@ -100,17 +109,8 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
return doWrite( collectionScope, entityId, new RowOp() {
@Override
public void doOp( final ColumnListMutation<UUID> colMutation ) {
-// try {
colMutation.putColumn( colName, getEntitySerializer()
.toByteBuffer( new EntityWrapper( entity.getStatus(), entity.getEntity() ) ) );
-// }
-// catch ( Exception e ) {
-// // throw better exception if we can
-// if ( entity != null || entity.getEntity().get() != null ) {
-// throw new CollectionRuntimeException( entity, collectionScope, e );
-// }
-// throw e;
-// }
}
} );
}
@@ -152,46 +152,110 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
rowKeys.add( rowKey );
}
+ /**
+ * Our settings may mean we exceed our maximum thrift buffer size. If we do, we have to make multiple requests, not just one.
+ * Perform the calculations and the appropriate request patterns
+ *
+ */
+
+ final int maxEntityResultSizeInBytes = serializationFig.getMaxEntitySize() * entityIds.size();
- final Iterator<Row<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID>> latestEntityColumns;
+ //if we're less than 1, set the number of requests to 1
+ final int numberRequests = Math.max(1, maxEntityResultSizeInBytes / cassandraFig.getThriftBufferSize());
+ final int entitiesPerRequest = entityIds.size() / numberRequests;
- try {
- latestEntityColumns = keyspace.prepareQuery( columnFamily ).getKeySlice( rowKeys )
- .withColumnRange( maxVersion, null, false, 1 ).execute().getResult()
- .iterator();
+
+ final Scheduler scheduler;
+
+ //if it's a single request, run it on the same thread
+ if(numberRequests == 1){
+ scheduler = Schedulers.immediate();
}
- catch ( ConnectionException e ) {
- throw new CollectionRuntimeException( null, collectionScope, "An error occurred connecting to cassandra",
- e );
+ //if it's more than 1 request, run them on the I/O scheduler
+ else{
+ scheduler = Schedulers.io();
}
- final EntitySetImpl entitySetResults = new EntitySetImpl( entityIds.size() );
+ final EntitySetImpl entitySetResults = Observable.from( rowKeys )
+ //buffer our entities per request, then for that buffer, execute the query in parallel (if neccessary)
+ .buffer(entitiesPerRequest )
+ .parallel( new Func1<Observable<List<ScopedRowKey
+ <CollectionPrefixedKey<Id>>>>, Observable<Rows<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID>>>() {
+
+
+ @Override
+ public Observable<Rows<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID>> call(
+ final Observable<List<ScopedRowKey<CollectionPrefixedKey<Id>>>> listObservable ) {
+
+
+ //here, we execute our query then emit the items either in parallel, or on the current thread if we have more than 1 request
+ return listObservable.map( new Func1<List<ScopedRowKey<CollectionPrefixedKey<Id>>>,
+ Rows<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID>>() {
+
+
+ @Override
+ public Rows<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> call(
+ final List<ScopedRowKey<CollectionPrefixedKey<Id>>> scopedRowKeys ) {
+
+ try {
+ return keyspace.prepareQuery( columnFamily ).getKeySlice( rowKeys )
+ .withColumnRange( maxVersion, null, false,
+ 1 ).execute().getResult();
+ }
+ catch ( ConnectionException e ) {
+ throw new CollectionRuntimeException( null, collectionScope, "An error occurred connecting to cassandra",
+ e );
+ }
+ }
+ } );
- while ( latestEntityColumns.hasNext() ) {
- final Row<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> row = latestEntityColumns.next();
- final ColumnList<UUID> columns = row.getColumns();
- if ( columns.size() == 0 ) {
- continue;
}
+ }, scheduler )
- final Id entityId = row.getKey().getKey().getSubKey();
+ //reduce all the output into a single Entity set
+ .reduce( new EntitySetImpl( entityIds.size() ),
+ new Func2<EntitySetImpl, Rows<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID>, EntitySetImpl>() {
+ @Override
+ public EntitySetImpl call( final EntitySetImpl entitySet,
+ final Rows<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> rows ) {
- final Column<UUID> column = columns.getColumnByIndex( 0 );
+ final Iterator<Row<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID>> latestEntityColumns = rows.iterator();
- final MvccEntity parsedEntity =
- new MvccColumnParser( entityId, getEntitySerializer() ).parseColumn( column );
+ while ( latestEntityColumns.hasNext() ) {
+ final Row<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> row = latestEntityColumns.next();
- //we *might* need to repair, it's not clear so check before loading into result sets
- final MvccEntity maybeRepaired = repair.maybeRepair( collectionScope, parsedEntity );
+ final ColumnList<UUID> columns = row.getColumns();
- entitySetResults.addEntity( maybeRepaired );
- }
+ if ( columns.size() == 0 ) {
+ continue;
+ }
+
+ final Id entityId = row.getKey().getKey().getSubKey();
+
+ final Column<UUID> column = columns.getColumnByIndex( 0 );
+
+ final MvccEntity parsedEntity =
+ new MvccColumnParser( entityId, getEntitySerializer() ).parseColumn( column );
+
+ //we *might* need to repair, it's not clear so check before loading into result sets
+ final MvccEntity maybeRepaired = repair.maybeRepair( collectionScope, parsedEntity );
+
+ entitySet.addEntity( maybeRepaired );
+ }
+
+
+
+ return entitySet;
+ }
+ } ).toBlocking().last();
return entitySetResults;
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56ce7ce6/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
index b40243d..119fb6d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV1Impl.java
@@ -26,6 +26,8 @@ import java.util.UUID;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
@@ -69,8 +71,8 @@ public class MvccEntitySerializationStrategyV1Impl extends MvccEntitySerializati
@Inject
- public MvccEntitySerializationStrategyV1Impl( final Keyspace keyspace, final SerializationFig serializationFig ) {
- super( keyspace, serializationFig );
+ public MvccEntitySerializationStrategyV1Impl( final Keyspace keyspace, final SerializationFig serializationFig, final CassandraFig cassandraFig ) {
+ super( keyspace, serializationFig, cassandraFig );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56ce7ce6/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
index 923c399..cd46c1e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Impl.java
@@ -27,6 +27,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.exception.DataCorruptionException;
import org.apache.usergrid.persistence.collection.exception.EntityTooLargeException;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.astyanax.FieldBuffer;
import org.apache.usergrid.persistence.core.astyanax.FieldBufferBuilder;
import org.apache.usergrid.persistence.core.astyanax.FieldBufferParser;
@@ -72,8 +73,8 @@ public class MvccEntitySerializationStrategyV2Impl extends MvccEntitySerializati
@Inject
- public MvccEntitySerializationStrategyV2Impl( final Keyspace keyspace, final SerializationFig serializationFig ) {
- super( keyspace, serializationFig );
+ public MvccEntitySerializationStrategyV2Impl( final Keyspace keyspace, final SerializationFig serializationFig, final CassandraFig cassandraFig ) {
+ super( keyspace, serializationFig, cassandraFig );
entitySerializer = new EntitySerializer( serializationFig );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/56ce7ce6/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java
index a496519..5f633a1 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyV2Test.java
@@ -20,25 +20,25 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
-import java.lang.annotation.Annotation;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.UUID;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.safehaus.guicyfig.Bypass;
-import org.safehaus.guicyfig.Env;
-import org.safehaus.guicyfig.Option;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.exception.EntityTooLargeException;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.util.EntityHelper;
+import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -49,6 +49,7 @@ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.inject.Inject;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -58,21 +59,27 @@ public abstract class MvccEntitySerializationStrategyV2Test extends MvccEntitySe
@Inject
protected SerializationFig serializationFig;
+ @Inject
+ protected CassandraFig cassandraFig;
+
+
private int setMaxEntitySize;
@Before
- public void setUp(){
+ public void setUp() {
- setMaxEntitySize = serializationFig.getMaxEntitySize();
+ setMaxEntitySize = serializationFig.getMaxEntitySize();
}
+
@After
- public void tearDown(){
+ public void tearDown() {
SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", setMaxEntitySize + "" );
}
+
/**
* Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
*/
@@ -83,7 +90,7 @@ public abstract class MvccEntitySerializationStrategyV2Test extends MvccEntitySe
//this is the size it works out to be when serialized, we want to allow this size
- SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", 65535*10+"");
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", 65535 * 10 + "" );
final Entity entity = EntityHelper.generateEntity( setSize );
//now we have one massive, entity, save it and retrieve it.
@@ -106,7 +113,12 @@ public abstract class MvccEntitySerializationStrategyV2Test extends MvccEntitySe
getMvccEntitySerializationStrategy().loadDescendingHistory( context, id, version, 100 );
- assertLargeEntity( mvccEntity, loaded );
+ assertTrue( loaded.hasNext() );
+
+ final MvccEntity loadedEntity = loaded.next();
+
+ assertLargeEntity( mvccEntity, loadedEntity );
+
MvccEntity returned =
serializationStrategy.load( context, Collections.singleton( id ), version ).getEntity( id );
@@ -115,7 +127,6 @@ public abstract class MvccEntitySerializationStrategyV2Test extends MvccEntitySe
}
-
/**
* Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
*/
@@ -142,23 +153,77 @@ public abstract class MvccEntitySerializationStrategyV2Test extends MvccEntitySe
}
- protected void assertLargeEntity( final MvccEntity expected, final Iterator<MvccEntity> returned ) {
- assertTrue( returned.hasNext() );
+ /**
+ * Tests an entity with more than 65535 bytes worth of data is successfully stored and retrieved
+ */
+ @Test
+ public void largeEntityReadWrite() throws ConnectionException {
+
+ //this is the size it works out to be when serialized, we want to allow this size
+
+ //extreme edge case, we can only get 2 entities per call
+ final int thriftBuffer = cassandraFig.getThriftBufferSize();
- final MvccEntity loadedEntity = returned.next();
- assertLargeEntity( expected, loadedEntity );
- }
+ //we use 20, using 2 causes cassandra to OOM. We don't have a large enough instance running locally
- protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+ final int maxEntitySize = ( int ) ( ( thriftBuffer * .9 ) / 20 );
- org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
- EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
+ SetConfigTestBypass.setValueByPass( serializationFig, "getMaxEntitySize", maxEntitySize + "" );
+
+
+ final int size = 100;
+
+ final HashMap<Id, MvccEntity> entities = new HashMap<>( size );
+
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "parent" ), "tests" );
+
+
+ for ( int i = 0; i < size; i++ ) {
+ final Entity entity = EntityHelper.generateEntity( ( int ) (maxEntitySize*.4) );
+
+ //now we have one massive, entity, save it and retrieve it.
+
+ final Id id = entity.getId();
+ ValidationUtils.verifyIdentity( id );
+ final UUID version = UUIDGenerator.newTimeUUID();
+ final MvccEntity.Status status = MvccEntity.Status.COMPLETE;
+
+ final MvccEntity mvccEntity = new MvccEntityImpl( id, version, status, entity );
+
+
+ getMvccEntitySerializationStrategy().write( context, mvccEntity ).execute();
+
+ entities.put( id, mvccEntity );
+ }
+
+
+ //now load it, we ask for 100 and we only are allowed 2 per trip due to our max size constraints. Should all
+ //still load (note that users should not be encouraged to use this strategy, it's a bad idea!)
+ final EntitySet loaded =
+ getMvccEntitySerializationStrategy().load( context, entities.keySet(), UUIDGenerator.newTimeUUID() );
+
+ assertNotNull( "Entity set was loaded", loaded );
+
+
+ for ( Map.Entry<Id, MvccEntity> entry : entities.entrySet() ) {
+
+ final MvccEntity returned = loaded.getEntity( entry.getKey() );
+
+ assertLargeEntity( entry.getValue(), returned );
+ }
}
+ protected void assertLargeEntity( final MvccEntity expected, final MvccEntity returned ) {
+
+ org.junit.Assert.assertEquals( "The loaded entity should match the stored entity", expected, returned );
+
+ EntityHelper.verifyDeepEquals( expected.getEntity().get(), returned.getEntity().get() );
+ }
}
[18/19] incubator-usergrid git commit: Merge remote-tracking branch
'origin/two-dot-o' into USERGRID-250-buffer-size-fix
Posted by sf...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o' into USERGRID-250-buffer-size-fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b240c6f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b240c6f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b240c6f7
Branch: refs/heads/two-dot-o
Commit: b240c6f708796144dd9d85027d7a1db8a9a7f5f5
Parents: 56ce7ce 74866a5
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 21 13:34:56 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 21 13:34:56 2014 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/GeoIT.java | 404 +++++++++++++------
.../index/impl/EsEntityIndexBatchImpl.java | 66 +--
.../index/impl/EsEntityIndexImpl.java | 105 +++--
.../persistence/index/impl/EsProvider.java | 55 +--
.../persistence/index/impl/EsQueryVistor.java | 3 +-
.../index/impl/FailureMonitorImpl.java | 4 +-
.../persistence/index/impl/IndexingUtils.java | 51 +--
.../index/query/CandidateResults.java | 3 -
.../persistence/index/query/EntityResults.java | 4 +-
.../persistence/index/query/Identifier.java | 3 +-
.../usergrid/persistence/index/query/Query.java | 26 +-
.../apache/usergrid/rest/AbstractRestIT.java | 5 +-
.../collection/BrowserCompatibilityTest.java | 2 +-
.../collection/DuplicateNameIT.java | 2 +-
.../collection/activities/PutTest.java | 2 +-
.../collection/groups/GroupResourceIT.java | 240 +++++++++--
.../collection/paging/PagingEntitiesTest.java | 4 +-
.../collection/paging/PagingResourceIT.java | 14 +-
.../users/ConnectionResourceTest.java | 16 +-
.../collection/users/OwnershipResourceIT.java | 18 +-
.../collection/users/RetrieveUsersTest.java | 4 +-
.../applications/queries/AndOrQueryTest.java | 13 +-
.../queries/BadGrammarQueryTest.java | 2 +-
.../applications/queries/GeoPagingTest.java | 222 +++++++++-
.../applications/queries/MatrixQueryTests.java | 2 +-
.../rest/applications/queries/OrderByTest.java | 6 +-
.../usergrid/rest/test/resource/Connection.java | 9 +
.../rest/test/resource/TestContext.java | 66 ++-
.../rest/test/resource/ValueResource.java | 2 +
.../rest/test/resource/app/Application.java | 16 +-
.../rest/test/resource/app/Collection.java | 81 ++++
.../usergrid/rest/test/resource/app/Group.java | 56 +++
.../test/resource/app/GroupsCollection.java | 66 +++
.../usergrid/rest/test/resource/app/Role.java | 56 +++
.../rest/test/resource/app/RolesCollection.java | 64 +++
.../rest/test/resource/app/UsersCollection.java | 26 +-
36 files changed, 1336 insertions(+), 382 deletions(-)
----------------------------------------------------------------------
[09/19] incubator-usergrid git commit: Reverted removal of load
method in commit 8beab4489672e403462ecdc9bfe6f3061954c076. This should not be
removed
Posted by sf...@apache.org.
Reverted removal of load method in commit 8beab4489672e403462ecdc9bfe6f3061954c076. This should not be removed
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/eb91d13e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/eb91d13e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/eb91d13e
Branch: refs/heads/two-dot-o
Commit: eb91d13eed6deab5b5bb94ccb31add69a04c104a
Parents: bc0a900
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 20 09:56:22 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 20 09:56:22 2014 -0700
----------------------------------------------------------------------
.../mvcc/MvccEntitySerializationStrategy.java | 14 ++++++++-
.../mvcc/stage/delete/MarkCommit.java | 2 +-
.../serialization/impl/EntityRepairImpl.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 30 ++++++++++++++++++++
...vccEntitySerializationStrategyProxyImpl.java | 9 ++++++
...MvccEntitySerializationStrategyImplTest.java | 17 ++++++-----
6 files changed, 62 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eb91d13e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
index faa76a3..3c694c5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
@@ -56,9 +56,21 @@ public interface MvccEntitySerializationStrategy extends Migration {
*/
public EntitySet load( CollectionScope scope, Collection<Id> entityIds, UUID maxVersion);
+ /**
+ * Load a list, from highest to lowest of the entity with versions <= version up to maxSize elements
+ *
+ * @param context The context to persist the entity into
+ * @param entityId The entity id to load
+ * @param version The max version to seek from. I.E a stored version <= this argument
+ * @param fetchSize The maximum size to return. If you receive this size, there may be more versions to load.
+ *
+ * @return A list of entities up to max size ordered from max(UUID)=> min(UUID). The return value should be null
+ * safe and return an empty list when there are no matches
+ */
+ public Iterator<MvccEntity> load( CollectionScope context, Id entityId, UUID version, int fetchSize );
/**
- * Load a historical list of entities, from highest to lowest of the entity with versions <= version up to maxSize elements
+ * Load a historical list of entities, from highest to lowest of the entity with versions < version up to maxSize elements
*
* @param context The context to persist the entity into
* @param entityId The entity id to load
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eb91d13e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index fab904d..5bcb9f8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -129,7 +129,7 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
@Override
protected Iterator<MvccEntity> getIterator() {
Iterator<MvccEntity> entities =
- entityStrat.loadHistory( collectionScope, entityId, entity.getVersion(), 100 );
+ entityStrat.load( collectionScope, entityId, entity.getVersion(), 100 );
return entities;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eb91d13e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
index a01ffdb..a94c688 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
@@ -69,7 +69,7 @@ public class EntityRepairImpl implements EntityRepair {
partialEntities.add( targetEntity );
final Iterator<MvccEntity> results = mvccEntitySerializationStrategy
- .loadHistory( collectionScope, targetEntity.getId(), targetEntity.getVersion(),
+ .load( collectionScope, targetEntity.getId(), targetEntity.getVersion(),
serializationFig.getBufferSize() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eb91d13e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index c964b68..1ec027f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -209,6 +209,36 @@ public abstract class MvccEntitySerializationStrategyImpl implements MvccEntityS
@Override
+ public Iterator<MvccEntity> load( final CollectionScope collectionScope, final Id entityId, final UUID version,
+ final int fetchSize ) {
+
+ Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
+ Preconditions.checkNotNull( entityId, "entity id is required" );
+ Preconditions.checkNotNull( version, "version is required" );
+ Preconditions.checkArgument( fetchSize > 0, "max Size must be greater than 0" );
+
+
+ final Id applicationId = collectionScope.getApplication();
+ final Id ownerId = collectionScope.getOwner();
+ final String collectionName = collectionScope.getName();
+
+ final CollectionPrefixedKey<Id> collectionPrefixedKey =
+ new CollectionPrefixedKey<>( collectionName, ownerId, entityId );
+
+
+ final ScopedRowKey<CollectionPrefixedKey<Id>> rowKey =
+ ScopedRowKey.fromKey( applicationId, collectionPrefixedKey );
+
+
+ RowQuery<ScopedRowKey<CollectionPrefixedKey<Id>>, UUID> query =
+ keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( rowKey )
+ .withColumnRange( version, null, false, fetchSize );
+
+ return new ColumnNameIterator( query, new MvccColumnParser( entityId, entityJsonSerializer ), false );
+ }
+
+
+ @Override
public Iterator<MvccEntity> loadHistory( final CollectionScope collectionScope, final Id entityId,
final UUID version, final int fetchSize ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eb91d13e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
index f5cf642..a9e01b1 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyProxyImpl.java
@@ -95,6 +95,15 @@ public class MvccEntitySerializationStrategyProxyImpl implements MvccEntitySeria
}
+ @Override
+ public Iterator<MvccEntity> load( final CollectionScope context, final Id entityId, final UUID version,
+ final int fetchSize ) {
+ if ( isOldVersion() ) {
+ return previous.load( context, entityId, version, fetchSize );
+ }
+
+ return current.load( context, entityId, version, fetchSize );
+ }
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eb91d13e/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index bc7c783..4b5ea7a 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -402,7 +402,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
//now ask for up to 10 versions from the current version, we should get cleared, v2, v1
UUID current = UUIDGenerator.newTimeUUID();
- Iterator<MvccEntity> entities = serializationStrategy.loadHistory( context, id, current, 3 );
+ Iterator<MvccEntity> entities = serializationStrategy.load( context, id, current, 3 );
MvccEntity first = entities.next();
assertEquals( clearedV3, first);
@@ -422,7 +422,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
serializationStrategy.delete( context, id , version1 ).execute();
serializationStrategy.delete( context, id , version2 ).execute();
- entities = serializationStrategy.loadHistory( context, id, current, 3 );
+ entities = serializationStrategy.load( context, id, current, 3 );
first = entities.next();
assertEquals( clearedV3, first );
@@ -432,7 +432,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
serializationStrategy.delete( context, id , version3 ).execute();
- entities = serializationStrategy.loadHistory( context, id, current, 3 );
+ entities = serializationStrategy.load( context, id, current, 3 );
Assert.assertTrue( !entities.hasNext());
}
@@ -639,7 +639,7 @@ public abstract class MvccEntitySerializationStrategyImplTest {
@Test(expected = NullPointerException.class)
public void loadListParamContext() throws ConnectionException {
- serializationStrategy.loadHistory( null, new SimpleId( "test" ), UUIDGenerator.newTimeUUID(), 1 );
+ serializationStrategy.load( null, new SimpleId( "test" ), UUIDGenerator.newTimeUUID(), 1 );
}
@@ -647,8 +647,8 @@ public abstract class MvccEntitySerializationStrategyImplTest {
public void loadListParamEntityId() throws ConnectionException {
serializationStrategy
- .loadHistory( new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" ),
- null, UUIDGenerator.newTimeUUID(), 1 );
+ .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), null, UUIDGenerator.newTimeUUID(),
+ 1 );
}
@@ -656,15 +656,14 @@ public abstract class MvccEntitySerializationStrategyImplTest {
public void loadListParamVersion() throws ConnectionException {
serializationStrategy
- .loadHistory( new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" ),
- new SimpleId( "test" ), null, 1 );
+ .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ), null, 1 );
}
@Test(expected = IllegalArgumentException.class)
public void loadListParamSize() throws ConnectionException {
- serializationStrategy.loadHistory( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ),
+ serializationStrategy.load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ),
UUIDGenerator.newTimeUUID(), 0 );
}
[03/19] incubator-usergrid git commit: Added new field buffer for
storing large data elements in Cassandra
Posted by sf...@apache.org.
Added new field buffer for storing large data elements in Cassandra
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/39b15765
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/39b15765
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/39b15765
Branch: refs/heads/two-dot-o
Commit: 39b157651b1316d7e1c76e592f5a2774a775f7f6
Parents: 3073fe8
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 19 14:11:30 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 19 14:11:30 2014 -0700
----------------------------------------------------------------------
.../persistence/core/astyanax/FieldBuffer.java | 70 ++++++++++
.../core/astyanax/FieldBufferBuilder.java | 87 ++++++++++++
.../core/astyanax/FieldBufferParser.java | 81 +++++++++++
.../core/astyanax/FieldBufferSerializer.java | 134 +++++++++++++++++++
.../astyanax/FieldBufferSerializerTest.java | 126 +++++++++++++++++
5 files changed, 498 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
new file mode 100644
index 0000000..898bc73
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+
+/**
+ * A utility class for storing multiple fields
+ */
+public class FieldBuffer {
+
+ private final List<ByteBuffer> fields;
+
+
+ /**
+ * Allocate a new field buffer with the expected max size. This allows us to pre-allocate
+ * our buffer for fields
+ *
+ * @param expectedMax
+ */
+ public FieldBuffer(final int expectedMax){
+ fields = new ArrayList<>( expectedMax );
+ }
+
+
+ /**
+ * Add the field and the serializer to the list
+ * @param value The serialized value to add to the buffer
+ */
+ public void add(ByteBuffer value){
+ //Note that we're not validating our byte buffer length. Since the length of a byte buffer IS an integer
+ //we can't possibly overflow an integer because they're the same data type
+ fields.add(value.duplicate() );
+ }
+
+
+ /**
+ * Return the list of all fields in read only format
+ * @return
+ */
+ public List<ByteBuffer> getFields(){
+ return Collections.unmodifiableList(fields);
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
new file mode 100644
index 0000000..88cfb62
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import com.netflix.astyanax.serializers.ByteBufferSerializer;
+import com.netflix.astyanax.serializers.ByteSerializer;
+import com.netflix.astyanax.serializers.BytesArraySerializer;
+import com.netflix.astyanax.serializers.IntegerSerializer;
+
+
+/**
+ * A builder pattern
+ */
+public class FieldBufferBuilder {
+
+
+ private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
+ private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+ private static final ByteSerializer BYTE_SERIALIZER = ByteSerializer.get();
+
+ private final FieldBuffer buffer;
+
+
+ public FieldBufferBuilder( final int elementSize ) {
+ buffer = new FieldBuffer( elementSize );
+ }
+
+
+ /**
+ * Add the integer to the fields.
+ * @param value
+ * @return
+ */
+ public FieldBufferBuilder addInteger( final int value ) {
+ buffer.add( INTEGER_SERIALIZER.toByteBuffer( value ) );
+ return this;
+ }
+
+
+ /**
+ * Add the byte array to the fieldbuilder
+ * @param bytes
+ * @return
+ */
+ public FieldBufferBuilder addBytes( final byte[] bytes ) {
+ buffer.add( BYTES_ARRAY_SERIALIZER.toByteBuffer( bytes ) );
+ return this;
+ }
+
+
+ /**
+ * Add the bytes to the fieldBuilder
+ * @param newByte
+ * @return
+ */
+ public FieldBufferBuilder addByte( final byte newByte ) {
+ buffer.add( BYTE_SERIALIZER.toByteBuffer( newByte ) );
+ return this;
+ }
+
+
+ /**
+ * Return the field buffer from the builder
+ * @return
+ */
+ public FieldBuffer build(){
+ return buffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
new file mode 100644
index 0000000..ef3b962
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import com.netflix.astyanax.serializers.ByteSerializer;
+import com.netflix.astyanax.serializers.BytesArraySerializer;
+import com.netflix.astyanax.serializers.IntegerSerializer;
+
+
+/**
+ * A parser for our field buffer
+ */
+public class FieldBufferParser {
+
+
+ private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
+ private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+ private static final ByteSerializer BYTE_SERIALIZER = ByteSerializer.get();
+
+ private final Iterator<ByteBuffer> fields;
+
+
+ public FieldBufferParser( final FieldBuffer buffer ) {
+ fields = buffer.getFields().iterator();
+ }
+
+
+ /**
+ * Return the value as an integer
+ */
+ public int getInteger() {
+ return INTEGER_SERIALIZER.fromByteBuffer( getNext() );
+ }
+
+
+ /**
+ * Return the value as a byte array
+ */
+ public byte[] getBytes() {
+ return BYTES_ARRAY_SERIALIZER.fromByteBuffer( getNext() );
+ }
+
+
+ /**
+ * return the next vlaue as a byte
+ */
+ public byte getByte() {
+ return BYTE_SERIALIZER.fromByteBuffer( getNext() );
+ }
+
+
+ private ByteBuffer getNext() {
+ if ( !fields.hasNext() ) {
+ throw new NoSuchElementException( "No more elements to return" );
+ }
+
+ return fields.next();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
new file mode 100644
index 0000000..d865349
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import com.netflix.astyanax.connectionpool.exceptions.SerializationException;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+
+
+/**
+ * Serializer that tests serializing field buggers
+ */
+public class FieldBufferSerializer extends AbstractSerializer<FieldBuffer> {
+
+
+ private static final FieldBufferSerializer INSTANCE = new FieldBufferSerializer();
+
+ private static final byte VERSION = 0;
+
+
+ public static FieldBufferSerializer get() {
+ return INSTANCE;
+ }
+
+
+ @Override
+ public ByteBuffer toByteBuffer( final FieldBuffer obj ) {
+ //create an output stream
+ final List<ByteBuffer> fields = obj.getFields();
+ final int fieldCount = fields.size();
+
+ //we start with 9. 1 byte for version, 4 bytes for total length, 4 bytes for the field count
+ int totalLength = 9;
+
+ for ( ByteBuffer fieldData : fields ) {
+ final int bufferLength = fieldData.remaining();
+ totalLength += 4 + bufferLength;
+ }
+
+ //now we have our total length, allocate it.
+
+ ByteBuffer buffer = ByteBuffer.allocate( totalLength );
+
+ buffer.put( VERSION );
+ buffer.putInt( totalLength );
+ buffer.putInt( fieldCount );
+
+ for ( ByteBuffer fieldData : fields ) {
+ final int bufferLength = fieldData.limit();
+
+ buffer.putInt( bufferLength );
+ buffer.put( fieldData );
+ }
+
+ buffer.rewind();
+ return buffer;
+ }
+
+
+ @Override
+ public FieldBuffer fromByteBuffer( final ByteBuffer byteBuffer ) {
+
+ final int totalSize = byteBuffer.remaining();
+ final byte version = byteBuffer.get();
+
+
+ //not what we expected, throw an ex
+ if ( version != VERSION ) {
+ throw new SerializationException(
+ "Unable to de-serialze. Expected version " + VERSION + " but was version " + version );
+ }
+
+ final int expectedTotalSize = byteBuffer.getInt();
+
+ if ( totalSize != expectedTotalSize ) {
+ throw new SerializationException(
+ "The total size we expected was different. Stored total size is " + expectedTotalSize
+ + " but actual buffer size is " + totalSize );
+ }
+
+
+ final int numberFields = byteBuffer.getInt();
+
+ final FieldBuffer buffer = new FieldBuffer( numberFields );
+
+
+ for ( int i = 0; i < numberFields; i++ ) {
+ final int bufferLength = byteBuffer.getInt();
+
+ final int startPosition = byteBuffer.position();
+ final int newLimit = startPosition + bufferLength;
+
+ //now read in the length and add it to our fieldBuffer
+
+
+ //we do this so we don't actually copy the underlying buff again. Instead we duplicate it
+ //and change our start and limits, to ensure we only read the field data when parsing
+ final ByteBuffer fieldData = byteBuffer.duplicate();
+
+ //set the limit that's the end of the field on the duplicate so that we won't read beyond this value
+ fieldData.limit( newLimit );
+
+ //advance our own internal buffer so that we can read the next field
+ byteBuffer.position(newLimit);
+
+
+
+ buffer.add( fieldData );
+ }
+
+
+ return buffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
new file mode 100644
index 0000000..e9a8194
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+
+/**
+ * Tests the builder, parser, and serialization of the entities
+ */
+public class FieldBufferSerializerTest {
+
+ /**
+ * Perform a very simple serialization of the 3 field types
+ */
+ @Test
+ public void simpleSerializer() {
+
+ final byte setByte = 1;
+ final int setInteger = 200;
+ final byte[] setByteArray = generateByteArray( 1000 );
+
+ FieldBufferBuilder builder = new FieldBufferBuilder( 3 );
+
+ builder.addByte( setByte );
+ builder.addInteger( setInteger );
+ builder.addBytes( setByteArray );
+
+
+ final FieldBufferSerializer serializer = new FieldBufferSerializer();
+
+ final ByteBuffer serialized = serializer.toByteBuffer( builder.build() );
+
+
+ final FieldBuffer parsed = serializer.fromByteBuffer( serialized );
+
+ FieldBufferParser parser = new FieldBufferParser( parsed );
+
+ final byte returnedByte = parser.getByte();
+
+ assertEquals( "Bytes should be equal", setByte, returnedByte );
+
+ final int returnedInt = parser.getInteger();
+
+ assertEquals( "Integer should be equal", setInteger, returnedInt );
+
+ final byte[] returnedByteArray = parser.getBytes();
+
+ assertArrayEquals( "arrays should be equal", setByteArray, returnedByteArray );
+ }
+
+
+ @Test
+ public void largerThanUnsignedShorts() {
+ final int maxShortSize = 65535;
+
+ final int lengthOfArray = maxShortSize + 1;
+
+ final byte setByte = 2;
+ final int setInteger = 400;
+
+
+ final byte[] setByteArray = generateByteArray( lengthOfArray );
+
+ FieldBufferBuilder builder = new FieldBufferBuilder( 1 );
+
+ builder.addBytes( setByteArray );
+ builder.addByte( setByte );
+ builder.addInteger( setInteger );
+
+ final FieldBufferSerializer serializer = new FieldBufferSerializer();
+
+ final ByteBuffer serialized = serializer.toByteBuffer( builder.build() );
+
+
+ final FieldBuffer parsed = serializer.fromByteBuffer( serialized );
+
+ FieldBufferParser parser = new FieldBufferParser( parsed );
+
+ final byte[] returnedArray = parser.getBytes();
+
+ assertArrayEquals( setByteArray, returnedArray );
+
+ final byte returnedByte = parser.getByte();
+
+ assertEquals( "Bytes should be equal", setByte, returnedByte );
+
+ final int returnedInt = parser.getInteger();
+
+ assertEquals( "Integer should be equal", setInteger, returnedInt );
+ }
+
+
+ private byte[] generateByteArray( final int length ) {
+ final byte[] data = new byte[length];
+
+ //set it to something other than 0 so we can be sure we're allocating correctly on parse
+ Arrays.fill( data, ( byte ) 1 );
+
+ return data;
+ }
+}
[08/19] incubator-usergrid git commit: Fixed wiring problem with new
proxy impl
Posted by sf...@apache.org.
Fixed wiring problem with new proxy impl
Took George's advice and refactored resource to external file to keep test cleaner
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bc0a9005
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bc0a9005
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bc0a9005
Branch: refs/heads/two-dot-o
Commit: bc0a900563249afa9c857b0401264e961d6e4458
Parents: 8beab44
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 19 17:51:13 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 19 17:51:13 2014 -0700
----------------------------------------------------------------------
stack/core/pom.xml | 32 +-
.../corepersistence/CpEntityDeleteListener.java | 3 +-
.../usergrid/persistence/LargeEntityIT.java | 513 +------
stack/core/src/test/resources/largeentity.json | 1329 ++++++++++++++++++
.../collection/util/EntityHelper.java | 2 +-
5 files changed, 1364 insertions(+), 515 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bc0a9005/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 98c7628..61ce099 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -142,22 +142,30 @@
<include>**/*.sh</include>
<include>**/stack.json</include>
<include>**/*.properties</include>
- <include>**/*.xml</include>
+ <include>**/*.xml</include>
+
</includes>
</resource>
</resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- <filtering>true</filtering>
- <includes>
- <include>**/*.yaml</include>
- <include>**/*.properties</include>
- <include>**/*.xml</include>
- </includes>
- </testResource>
- </testResources>
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>**/*.yaml</include>
+ <include>**/*.properties</include>
+ <include>**/*.xml</include>
+ </includes>
+ </testResource>
+ <testResource>
+ <directory>src/test/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>largeentity.json</include>
+ </includes>
+ </testResource>
+ </testResources>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bc0a9005/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
index 4ec4056..8785ab2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
@@ -30,6 +30,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityDeleteEvent;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.entity.EntityVersion;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import com.google.inject.Inject;
@@ -54,7 +55,7 @@ public class CpEntityDeleteListener {
private final SerializationFig serializationFig;
@Inject
- public CpEntityDeleteListener(final MvccEntitySerializationStrategy entityMetadataSerialization,
+ public CpEntityDeleteListener(@ProxyImpl final MvccEntitySerializationStrategy entityMetadataSerialization,
final Keyspace keyspace,
final SerializationFig serializationFig){
this.entityMetadataSerialization = entityMetadataSerialization;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bc0a9005/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
index 0954ded..ad6ca27 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/LargeEntityIT.java
@@ -19,6 +19,10 @@
package org.apache.usergrid.persistence;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.Map;
import org.junit.Rule;
@@ -51,7 +55,13 @@ public class LargeEntityIT extends AbstractCoreIT {
LOG.debug( "testLargeEntityCrud" );
- final Map<String, Object> json = ( Map<String, Object> ) JsonUtils.parse( TEST_DATA );
+ final URL resource = this.getClass().getClassLoader().getResource( TEST_DATA_FILE );
+
+ final byte[] fileData = Files.readAllBytes( Paths.get( resource.toURI() ) );
+
+ final String fileAsString = new String( fileData, Charset.forName( "UTF-8" ) );
+
+ final Map<String, Object> json = ( Map<String, Object> ) JsonUtils.parse( fileAsString );
final EntityManager em = app.getEntityManager();
@@ -68,504 +78,5 @@ public class LargeEntityIT extends AbstractCoreIT {
}
- private static final String TEST_DATA =
- "{\n" + " \"name\": \"p333812236\",\n" + " \"catentryid\": 7176,\n" + " \"productid\": 33381,\n"
- + " \"services\": [\n" + " {\n" + " \"id\": 575,\n" + " \"name\": \"Monogram\",\n"
- + " \"required\": 0,\n" + " \"attributes\": [\n" + " {\n"
- + " \"id\": 80224,\n" + " \"name\": \"Style/Case\",\n"
- + " \"value\": \"Block Uppercase\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 3,\n" + " \"label\": \"\",\n" + " \"maxlength\": 10,\n"
- + " \"additionalvalidchars\": \"33,34,35,38,39,40,41,43,44,45,46,47,63,64\",\n"
- + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 80225,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Block Upper- and Lowercase\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"label\": \"\",\n"
- + " \"maxlength\": 10,\n"
- + " \"additionalvalidchars\": \"33,34,35,38,39,40,41,43,44,45,46,47,63,64\",\n"
- + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 80234,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Times Bold Uppercase\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"label\": \"\",\n"
- + " \"maxlength\": 10,\n"
- + " \"additionalvalidchars\": \"33,36,37,38,39,40,41,44,45,46,47,58,59,63\",\n"
- + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 80235,\n"
- + " \"name\": \"Style/Case\",\n"
- + " \"value\": \"Times Bold Upper- and Lowercase\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 3,\n" + " \"label\": \"\",\n" + " \"maxlength\": 10,\n"
- + " \"additionalvalidchars\": \"33,36,37,38,39,40,41,44,45,46,47,58,59,63\",\n"
- + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 80237,\n"
- + " \"name\": \"Monogram\",\n" + " \"value\": \"text\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 1,\n" + " \"label\": \"\",\n"
- + " \"maxlength\": 10,\n" + " \"notes\": \"\"\n" + " },\n" + " {\n"
- + " \"id\": 137788,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Black\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137789,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Navy\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137790,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Royal Blue\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137791,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Brown\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137792,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Purple\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137793,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Red\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137794,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Sea Green\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137795,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Hunter Green\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137796,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Dark Green\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137797,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Gold\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137798,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"White\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137799,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Platinum\",\n" + " \"isdefault\": 1,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137800,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Natural\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137801,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Taupe\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137802,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Light Purple\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137803,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Burgundy\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137804,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Teal\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137805,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Mimosa Yellow\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137806,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Turkey Red\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137807,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Lucerne Blue\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137808,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Nickel\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137809,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Orange\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137810,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Standard Blue\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137811,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Cobalt\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137812,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Dark Pink\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137813,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Sage\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137814,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Salmon\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137815,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Lime\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137816,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Rancho Rose\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137817,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Olive\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137818,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Rose\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137819,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Nile Green\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137820,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Dark Khaki\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137821,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Eggplant\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137822,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Cypress\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137823,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Terra Cotta\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137824,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Walnut\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137825,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Burlap\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 137826,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Loden\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 210983,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Copper\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 210984,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Dark Teal\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 216129,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Pink\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 227825,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Dark Brown\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 227826,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Emerald\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 227827,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Mid Blue\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 291205,\n" + " \"name\": \"Style/Case\",\n"
- + " \"value\": \"Script Upper/Lower\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 3,\n" + " \"label\": \"\",\n" + " \"maxlength\": 10,\n"
- + " \"additionalvalidchars\": \"33,34,35,39,43,44,46,47,63\",\n"
- + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 291206,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Thin Script Upper/Lower\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"label\": \"\",\n"
- + " \"maxlength\": 10,\n"
- + " \"additionalvalidchars\": \"33,34,35,38,39,43,44,45,46,47,63\",\n"
- + " \"notes\": \"\"\n" + " },\n" + " {\n" + " \"id\": 291207,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Athletic Upper\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"label\": \"\",\n"
- + " \"maxlength\": 10,\n" + " \"notes\": \"\"\n" + " },\n" + " {\n"
- + " \"id\": 353932,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Isle Green\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " },\n" + " {\n"
- + " \"id\": 368998,\n" + " \"name\": \"Thread Color\",\n"
- + " \"value\": \"Plum\",\n" + " \"isdefault\": 0,\n"
- + " \"sequence\": 4,\n" + " \"maxlength\": 0\n" + " }\n" + " ]\n"
- + " },\n" + " {\n" + " \"id\": 48071,\n"
- + " \"name\": \"Monogram Single Large Letter\",\n" + " \"required\": 0,\n"
- + " \"attributes\": [\n" + " {\n" + " \"id\": 291208,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Times Single Upper\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 1\n"
- + " },\n" + " {\n" + " \"id\": 291209,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Heirloom Single Upper\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 1\n"
- + " },\n" + " {\n" + " \"id\": 291210,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Athletic Single Upper\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 1\n"
- + " },\n" + " {\n" + " \"id\": 291211,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Black\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291212,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Brown\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291213,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Burgundy\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291214,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Burlap\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291215,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Cobalt\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291216,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Copper\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291217,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Cypress\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291218,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Brown\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291219,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291220,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Khaki\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291221,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Pink\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291222,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Teal\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291223,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Eggplant\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291224,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Emerald\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291225,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Gold\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291226,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Hunter Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291227,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Light Purple\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291228,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Lime\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291229,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Loden\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291230,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Lucerne Blue\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291231,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Mid Blue\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291232,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Mimosa Yellow\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291233,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Natural\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291234,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Navy\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291235,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Nickel\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291236,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Nile Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291237,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Olive\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291238,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Orange\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291239,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Pink\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291240,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Platinum\",\n"
- + " \"isdefault\": 1,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291241,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Purple\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291242,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Rancho Rose\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291243,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Red\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291244,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Rose\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291245,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Royal Blue\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291246,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Sage\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291247,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Salmon\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291248,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Sea Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291249,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Standard Blue\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291250,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Taupe\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291251,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Teal\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291252,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Terra Cotta\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291253,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Turkey Red\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291254,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Walnut\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291255,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"White\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291256,\n"
- + " \"name\": \"Monogram Single Line 1\",\n" + " \"value\": \"text\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 1,\n" + " \"maxlength\": 1\n"
- + " },\n" + " {\n" + " \"id\": 353933,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Isle Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 368999,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Plum\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " }\n" + " ]\n" + " },\n" + " {\n" + " \"id\": 48072,\n"
- + " \"name\": \"Monogram 3-Letter\",\n" + " \"required\": 0,\n"
- + " \"attributes\": [\n" + " {\n" + " \"id\": 291257,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Block Stack\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 1\n"
- + " },\n" + " {\n" + " \"id\": 291258,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Flared\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 3\n"
- + " },\n" + " {\n" + " \"id\": 291259,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Classic\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 3\n"
- + " },\n" + " {\n" + " \"id\": 291260,\n"
- + " \"name\": \"Style/Case\",\n" + " \"value\": \"Insignia\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 3,\n" + " \"maxlength\": 3\n"
- + " },\n" + " {\n" + " \"id\": 291261,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Black\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291262,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Brown\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291263,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Burgundy\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291264,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Burlap\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291265,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Cobalt\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291266,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Copper\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291267,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Cypress\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291268,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Brown\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291269,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291270,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Khaki\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291271,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Pink\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291272,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Dark Teal\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291273,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Eggplant\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291274,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Emerald\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291275,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Gold\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291276,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Hunter Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291277,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Light Purple\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291278,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Lime\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291279,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Loden\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291280,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Lucerne Blue\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291281,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Mid Blue\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291282,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Mimosa Yellow\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291283,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Natural\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291284,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Navy\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291285,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Nickel\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291286,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Nile Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291287,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Olive\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291288,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Orange\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291289,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Pink\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291290,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Platinum\",\n"
- + " \"isdefault\": 1,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291291,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Purple\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291292,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Rancho Rose\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291293,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Red\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291294,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Rose\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291295,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Royal Blue\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291296,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Sage\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291297,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Salmon\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291298,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Sea Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291299,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Standard Blue\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291300,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Taupe\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291301,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Teal\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291302,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Terra Cotta\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291303,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Turkey Red\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291304,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Walnut\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291305,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"White\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 291306,\n"
- + " \"name\": \"Monogram 3-Letter Line 1\",\n" + " \"value\": \"text\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 1,\n" + " \"maxlength\": 3\n"
- + " },\n" + " {\n" + " \"id\": 353934,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Isle Green\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " },\n" + " {\n" + " \"id\": 369000,\n"
- + " \"name\": \"Thread Color\",\n" + " \"value\": \"Plum\",\n"
- + " \"isdefault\": 0,\n" + " \"sequence\": 4,\n" + " \"maxlength\": 0\n"
- + " }\n" + " ]\n" + " },\n" + " {\n" + " \"id\": 3916,\n"
- + " \"name\": \"Gift Boxing\",\n" + " \"required\": 0,\n" + " \"attributes\": [\n"
- + " {\n" + " \"id\": 317426,\n" + " \"name\": \"Box Style\",\n"
- + " \"value\": \"Tan/Green Ribbon\",\n" + " \"isdefault\": 1,\n"
- + " \"sequence\": 1,\n" + " \"maxlength\": 0\n" + " }\n" + " ]\n"
- + " }\n" + " ]\n" + "}";
+ private static final String TEST_DATA_FILE = "largeentity.json";
}