You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2016/06/14 17:14:40 UTC
[01/44] usergrid git commit: Cherrypicking first of the changes to
account for future possible instances of the class cast exception
Repository: usergrid
Updated Branches:
refs/heads/usergrid-1268-akka-211 324ef35c9 -> a93cad8fa
Cherrypicking first of the changes to account for future possible instances of the class cast exception
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c46e1b68
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c46e1b68
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c46e1b68
Branch: refs/heads/usergrid-1268-akka-211
Commit: c46e1b6871516009edeb4ab1a35603850f3645d8
Parents: 6b195a0
Author: George Reyes <gr...@apache.org>
Authored: Fri May 6 15:36:11 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Fri May 6 15:36:11 2016 -0700
----------------------------------------------------------------------
.../persistence/entities/Notification.java | 37 +++++++++-----
.../impl/ApplicationQueueManagerImpl.java | 52 +++++++++++++++-----
.../AbstractServiceNotificationIT.java | 26 ++++------
3 files changed, 75 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c46e1b68/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
index d4f3529..bb2e03f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
@@ -17,18 +17,28 @@
package org.apache.usergrid.persistence.entities;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.*;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PathQuery;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.annotations.EntityCollection;
import org.apache.usergrid.persistence.annotations.EntityProperty;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.utils.InflectionUtils;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
/**
* The entity class for representing Notifications.
*/
@@ -112,7 +122,7 @@ public class Notification extends TypedEntity {
/** Map containing a count for "sent" and "errors" */
@EntityProperty
- protected Map<String, Long> statistics;
+ protected Map<String, Object> statistics;
@EntityProperty
protected Map<String, Object> filters;
@@ -275,11 +285,11 @@ public class Notification extends TypedEntity {
}
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
- public Map<String, Long> getStatistics() {
+ public Map<String, Object> getStatistics() {
return statistics;
}
- public void setStatistics(Map<String, Long> statistics) {
+ public void setStatistics(Map<String, Object> statistics) {
this.statistics = statistics;
}
@@ -294,13 +304,18 @@ public class Notification extends TypedEntity {
public void updateStatistics(long sent, long errors) {
if (this.statistics == null) {
- this.statistics = new HashMap<String, Long>(2);
+ this.statistics = new HashMap<String, Object>(2);
this.statistics.put("sent", sent);
this.statistics.put("errors", errors);
} else {
- this.statistics.put("sent", sent + this.statistics.get("sent"));
- this.statistics.put("errors",
- errors + this.statistics.get("errors"));
+ if(this.statistics.get( "sent" ) instanceof Integer){
+ this.statistics.put( "sent", sent + (Integer) this.statistics.get( "sent" ) );
+ this.statistics.put( "errors", errors + (Integer) this.statistics.get( "errors" ) );
+ }
+ else if (this.statistics.get( "sent" ) instanceof Long ) {
+ this.statistics.put( "sent", sent + (Long) this.statistics.get( "sent" ) );
+ this.statistics.put( "errors", errors + (Long) this.statistics.get( "errors" ) );
+ }
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c46e1b68/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index eb5d794..23b21f2 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -16,29 +16,55 @@
*/
package org.apache.usergrid.services.notifications.impl;
-import com.codahale.metrics.Meter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.batch.JobExecution;
-import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.core.executor.TaskExecutorFactory;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.entities.*;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PathQuery;
import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.entities.Device;
+import org.apache.usergrid.persistence.entities.Notification;
+import org.apache.usergrid.persistence.entities.Notifier;
+import org.apache.usergrid.persistence.entities.Receipt;
+import org.apache.usergrid.persistence.entities.User;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueMessage;
-import org.apache.usergrid.services.notifications.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.usergrid.services.notifications.ApplicationQueueManager;
+import org.apache.usergrid.services.notifications.ApplicationQueueMessage;
+import org.apache.usergrid.services.notifications.JobScheduler;
+import org.apache.usergrid.services.notifications.NotificationsService;
+import org.apache.usergrid.services.notifications.ProviderAdapter;
+import org.apache.usergrid.services.notifications.ProviderAdapterFactory;
+import org.apache.usergrid.services.notifications.TaskManager;
+import org.apache.usergrid.services.notifications.TaskTracker;
+
+import com.codahale.metrics.Meter;
+
import rx.Observable;
-import rx.Scheduler;
import rx.Subscriber;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicInteger;
public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c46e1b68/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
index 91b94b2..775827e 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
@@ -16,24 +16,18 @@
*/
package org.apache.usergrid.services.notifications;
-import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.entities.Notification;
-import org.apache.usergrid.persistence.entities.Receipt;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.services.ServiceManagerFactory;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PathQuery;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.entities.Notification;
+import org.apache.usergrid.persistence.entities.Receipt;
import org.apache.usergrid.services.AbstractServiceIT;
-import org.springframework.beans.factory.annotation.Autowired;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -106,17 +100,17 @@ public abstract class AbstractServiceNotificationIT extends AbstractServiceIT {
}
protected void checkStatistics(Notification notification, long sent, long errors) throws Exception{
- Map<String, Long> statistics = null;
+ Map<String, Object> statistics = null;
long timeout = System.currentTimeMillis() + 10000;
while (System.currentTimeMillis() < timeout) {
Thread.sleep(200);
statistics = app.getEntityManager().get(notification.getUuid(), Notification.class).getStatistics();
- if (statistics.get("sent")==sent && statistics.get("errors")==errors) {
+ if ((Long)statistics.get("sent")==sent && (Long)statistics.get("errors")==errors) {
break;
}
}
- assertEquals(sent, statistics.get("sent").longValue());
- assertEquals(errors, statistics.get("errors").longValue());
+ assertEquals(sent, ((Long)statistics.get("sent")).longValue());
+ assertEquals(errors, ((Long)statistics.get("errors")).longValue());
}
}
[03/44] usergrid git commit: Fixed error with a PUT call actually
sending a POST. Added additional assert to make sure that a POST will not
encounter this issue as well.
Posted by sn...@apache.org.
Fixed error with a PUT call actually sending a POST.
Added additional assert to make sure that a POST will not encounter this issue as well.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/987ffdc0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/987ffdc0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/987ffdc0
Branch: refs/heads/usergrid-1268-akka-211
Commit: 987ffdc01592b0b6e61b701720e04cfc8ba29331
Parents: 0f13650
Author: George Reyes <gr...@apache.org>
Authored: Mon May 9 10:50:36 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Mon May 9 10:50:36 2016 -0700
----------------------------------------------------------------------
.../collection/CollectionsResourceIT.java | 15 ++++++++----
.../resource/endpoints/CollectionEndpoint.java | 24 +++++++++++---------
2 files changed, 23 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/987ffdc0/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index 7afc87a..0908f5c 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -24,7 +24,6 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
-import java.util.UUID;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.ClientErrorException;
@@ -1112,7 +1111,6 @@ public class CollectionsResourceIT extends AbstractRestIT {
ApiResponse notifierNode = this.pathResource(getOrgAppPath("notifier")).post(ApiResponse.class,notifier);
- UUID uuid = UUIDUtils.newTimeUUID();
// create user
Map payloads = new HashMap<>( );
@@ -1132,14 +1130,21 @@ public class CollectionsResourceIT extends AbstractRestIT {
payload.put( "statistics",statistics);
-
-
- ApiResponse user = this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).put(null,payload );
+ this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).post(payload );
this.refreshIndex();
Collection user2 = this.app().collection("notifications").get();
assertEquals(1,user2.getNumOfEntities());
+ this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).put(null,payload );
+ this.refreshIndex();
+
+ user2 = this.app().collection("notifications").get();
+
+ assertEquals(2,user2.getNumOfEntities());
+
}
+
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/987ffdc0/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java
index ad5d9a6..d1c7ab4 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java
@@ -16,22 +16,24 @@
*/
package org.apache.usergrid.rest.test.resource.endpoints;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.usergrid.rest.test.resource.model.Collection;
-import org.apache.usergrid.rest.test.resource.state.ClientContext;
-import org.apache.usergrid.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.client.Entity;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.MediaType;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.rest.test.resource.model.Collection;
+import org.apache.usergrid.rest.test.resource.state.ClientContext;
+import org.apache.usergrid.utils.StringUtils;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
/**
* //myorg/myapp/mycollection
@@ -358,7 +360,7 @@ public class CollectionEndpoint extends NamedResource {
// use string type so we can log actual response from server
String responseString = resource.request()
.accept(acceptHeader)
- .post( javax.ws.rs.client.Entity.json( entity ), String.class);
+ .put( javax.ws.rs.client.Entity.json( entity ), String.class);
if (logger.isDebugEnabled()) {
logger.debug("Response from put: " + responseString);
[26/44] usergrid git commit: Ensure that field names are treated
consistently and in a case-insensitive way.
Posted by sn...@apache.org.
Ensure that field names are treated consistently and in a case-insensitive way.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5452d68c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5452d68c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5452d68c
Branch: refs/heads/usergrid-1268-akka-211
Commit: 5452d68c29bb59ec8794639cdc8d6d819792e8bb
Parents: 151abf7
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jun 1 09:56:01 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jun 1 09:56:01 2016 -0400
----------------------------------------------------------------------
.../read/search/CandidateEntityFilter.java | 13 +++
.../model/field/value/EntityObject.java | 15 ++-
.../queries/SelectMappingsQueryTest.java | 105 +++++++++++++++++++
3 files changed, 128 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5452d68c/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index 4aa6c8d..7770436 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -161,6 +161,13 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
}
+ /**
+ * Sets field in result map with support for nested fields via recursion.
+ *
+ * @param result The result map of filtered fields
+ * @param parts The parts of the field name (more than one if field is nested)
+ * @param fieldMap Map of fields of the object
+ */
private void nestedFieldSet( Map<String, Field> result, String[] parts, Map<String, Field> fieldMap) {
if ( parts.length > 0 ) {
@@ -184,6 +191,12 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
}
+ /**
+ * Check to see if field should be included in filtered result with support for nested fields via recursion.
+ *
+ * @param parts The parts of the field name (more than one if field is nested)
+ * @param fieldMap Map of fields of the object
+ */
private boolean nestedFieldCheck( String[] parts, Map<String, Field> fieldMap) {
if ( parts.length > 0 ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5452d68c/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java
index db44e87..a157029 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java
@@ -19,10 +19,7 @@
package org.apache.usergrid.persistence.model.field.value;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
import org.apache.usergrid.persistence.model.field.Field;
@@ -41,11 +38,19 @@ public class EntityObject implements Serializable {
@JsonIgnore
private long size;
+ // field names are treated in case-insensitive way by design
+ static class CaseInsensitiveComparator implements Comparator<String> {
+ public int compare(String o1, String o2) {
+ return o1.compareToIgnoreCase(o2);
+ }
+ }
+ public static final CaseInsensitiveComparator INSTANCE = new CaseInsensitiveComparator();
+
/**
* Fields the users can set
*/
@JsonTypeInfo( use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class" )
- private final Map<String, Field> fields = new HashMap<String, Field>();
+ private Map<String, Field> fields = new TreeMap<String, Field>(INSTANCE);
/**
* Add the field, return the old one if it existed
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5452d68c/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
new file mode 100644
index 0000000..eb6aeee
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.applications.queries;
+
+import org.apache.usergrid.rest.test.resource.model.Collection;
+import org.apache.usergrid.rest.test.resource.model.Entity;
+import org.apache.usergrid.rest.test.resource.model.QueryParameters;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+
+public class SelectMappingsQueryTest extends QueryTestBase {
+ private static final Logger logger = LoggerFactory.getLogger(OrderByTest.class);
+
+
+ @Test
+ public void testNestedSelectFieldNames() throws Exception {
+
+ generateTestEntities(20, "things");
+
+ QueryParameters params = new QueryParameters()
+ .setQuery("select actor.displayName,sometestprop where sometestprop = 'testprop'");
+ Collection things = this.app().collection("things").get(params);
+ assertEquals( 10, things.getNumOfEntities() );
+
+ Iterator<Entity> iter = things.iterator();
+ while ( iter.hasNext() ) {
+
+ Entity entity = iter.next();
+ assertEquals( 5, entity.getDynamicProperties().size() );
+
+ assertNotNull( entity.getDynamicProperties().get("uuid") );
+ assertNotNull( entity.getDynamicProperties().get("type") );
+ assertNotNull( entity.getDynamicProperties().get("metadata") );
+ assertNotNull( entity.getDynamicProperties().get("sometestprop") );
+
+ Map<String, Object> actor = (Map<String, Object>)entity.getDynamicProperties().get("actor");
+ assertNotNull( actor );
+ assertNotNull( actor.get("displayName") );
+
+ }
+ }
+
+
+ /**
+ * Shows that field names are case-insensitive.
+ * If you define two fields with same name but different cases, behavior is undefined.
+ */
+ @Test
+ public void testFieldNameCaseSensitivity() throws Exception {
+
+ int numberOfEntities = 10;
+ String collectionName = "things";
+
+ Entity[] entities = new Entity[numberOfEntities];
+ Entity props = new Entity();
+
+ for (int i = 0; i < numberOfEntities; i++) {
+ props.put("testProp", "a");
+ props.put("testprop", "b");
+ entities[i] = app().collection(collectionName).post(props);
+ }
+ refreshIndex();
+
+ {
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testProp = 'b'" );
+ Collection things = this.app().collection( "things" ).get( params );
+
+ // if field names were case sensitive, this would fail
+ assertEquals( numberOfEntities, things.getNumOfEntities() );
+ }
+
+ {
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testprop='b'" );
+ Collection things = this.app().collection( "things" ).get( params );
+
+ assertEquals( numberOfEntities, things.getNumOfEntities() );
+ }
+
+ }
+
+}
[13/44] usergrid git commit: Minor refactoring and formatting changes.
Posted by sn...@apache.org.
Minor refactoring and formatting changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e9e7c39c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e9e7c39c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e9e7c39c
Branch: refs/heads/usergrid-1268-akka-211
Commit: e9e7c39c46ca0b49b5891ee04bdac648e9777cee
Parents: 0b41620
Author: Dave Johnson <sn...@apache.org>
Authored: Fri May 20 16:00:21 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri May 20 16:00:21 2016 -0400
----------------------------------------------------------------------
.../pipeline/read/search/CandidateEntityFilter.java | 12 +++++++++---
.../java/org/apache/usergrid/persistence/IndexIT.java | 2 +-
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e9e7c39c/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index e02e6c7..4aa6c8d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -163,17 +163,19 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
private void nestedFieldSet( Map<String, Field> result, String[] parts, Map<String, Field> fieldMap) {
if ( parts.length > 0 ) {
+
if ( fieldMap.containsKey( parts[0] )) {
Field field = fieldMap.get( parts[0] );
if ( field instanceof EntityObjectField ) {
EntityObjectField eof = (EntityObjectField)field;
- if ( result.get( parts[0] ) == null ) {
- result.put( parts[0], new EntityObjectField( parts[0], new EntityObject() ) );
- }
+ result.putIfAbsent( parts[0], new EntityObjectField( parts[0], new EntityObject() ) );
+
+ // recursion
nestedFieldSet(
((EntityObjectField)result.get( parts[0] )).getValue().getFieldMap(),
Arrays.copyOfRange(parts, 1, parts.length),
eof.getValue().getFieldMap());
+
} else {
result.put( parts[0], field );
}
@@ -184,11 +186,15 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
private boolean nestedFieldCheck( String[] parts, Map<String, Field> fieldMap) {
if ( parts.length > 0 ) {
+
if ( fieldMap.containsKey( parts[0] )) {
Field field = fieldMap.get( parts[0] );
if ( field instanceof EntityObjectField ) {
EntityObjectField eof = (EntityObjectField)field;
+
+ // recursion
return nestedFieldCheck( Arrays.copyOfRange(parts, 1, parts.length), eof.getValue().getFieldMap());
+
} else {
return true;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e9e7c39c/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index aaf4c33..d62f88e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -528,7 +528,7 @@ public class IndexIT extends AbstractCoreIT {
assertTrue( first.getDynamicProperties().size() == 3 );
}
- // multiple nested fields with one doubly-nesed field
+ // multiple nested fields with one doubly-nested field
{
Query query = Query.fromQL( "select data.rando, data.mondo, data.misc.range where status = 'pickled'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
[35/44] usergrid git commit: USERGRID-1295 - Re-introduce a more
efficient de-index upon entity delete and entity updates. Remove the
inefficient code as a safety measure so it can't be used again.
Posted by sn...@apache.org.
USERGRID-1295 - Re-introduce a more efficient de-index upon entity delete and entity updates. Remove the inefficient code as a safety measure so it can't be used again.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7143cbaf
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7143cbaf
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7143cbaf
Branch: refs/heads/usergrid-1268-akka-211
Commit: 7143cbaf6c26b4db00d14fb3a1b9e3eb8a2e068e
Parents: 66bb5cd
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jun 3 22:18:12 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jun 3 22:18:12 2016 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../corepersistence/EntityManagerFig.java | 2 +-
.../asyncevents/AsyncEventService.java | 6 +-
.../asyncevents/AsyncEventServiceImpl.java | 11 +-
.../asyncevents/EventBuilder.java | 6 +-
.../asyncevents/EventBuilderImpl.java | 46 ++++---
.../model/DeIndexOldVersionsEvent.java | 12 +-
.../corepersistence/index/IndexService.java | 23 +++-
.../corepersistence/index/IndexServiceImpl.java | 86 +++++++------
.../collection/EntityCollectionManager.java | 10 +-
.../impl/EntityCollectionManagerImpl.java | 32 ++++-
.../serialization/SerializationFig.java | 5 +
.../serialization/impl/LogEntryIterator.java | 128 +++++++++++++++++++
.../usergrid/persistence/index/EntityIndex.java | 5 +-
.../usergrid/persistence/index/IndexFig.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 64 +++-------
.../persistence/index/impl/EntityIndexTest.java | 41 ------
17 files changed, 313 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 68f5d71..fd31cf6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -610,7 +610,7 @@ public class CpEntityManager implements EntityManager {
// queue up an event to clean-up older versions than this one from the index
if (entityManagerFig.getDeindexOnUpdate()) {
- indexService.queueDeIndexOldVersion( applicationScope, entityId );
+ indexService.queueDeIndexOldVersion( applicationScope, cpEntity.getId(), cpEntity.getVersion());
}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 4c50aee..655a968 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -39,6 +39,6 @@ public interface EntityManagerFig extends GuicyFig {
int sleep();
@Key( "usergrid.entityManager.enable_deindex_on_update" )
- @Default( "false" )
+ @Default( "true" )
boolean getDeindexOnUpdate();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index 9f34604..d833cf7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -27,6 +27,8 @@ import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import java.util.UUID;
+
/**
* Low level queue service for events in the entity. These events are fire and forget, and will always be asynchronous
@@ -83,11 +85,11 @@ public interface AsyncEventService extends ReIndexAction {
void queueIndexOperationMessage( final IndexOperationMessage indexOperationMessage );
/**
- *
* @param applicationScope
* @param entityId
+ * @param markedVersion
*/
- void queueDeIndexOldVersion(final ApplicationScope applicationScope, final Id entityId);
+ void queueDeIndexOldVersion(final ApplicationScope applicationScope, final Id entityId, UUID markedVersion);
/**
* current queue depth
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 3b01292..fa175ab 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -584,11 +584,11 @@ public class AsyncEventServiceImpl implements AsyncEventService {
@Override
- public void queueDeIndexOldVersion(final ApplicationScope applicationScope, final Id entityId) {
+ public void queueDeIndexOldVersion(final ApplicationScope applicationScope, final Id entityId, UUID markedVersion) {
// queue the de-index of old versions to the topic so cleanup happens in all regions
offerTopic( new DeIndexOldVersionsEvent( queueFig.getPrimaryRegion(),
- new EntityIdScope( applicationScope, entityId)) );
+ new EntityIdScope( applicationScope, entityId), markedVersion) );
}
@@ -596,10 +596,11 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public IndexOperationMessage handleDeIndexOldVersionEvent ( final DeIndexOldVersionsEvent deIndexOldVersionsEvent){
- ApplicationScope applicationScope = deIndexOldVersionsEvent.getEntityIdScope().getApplicationScope();
- Id entityId = deIndexOldVersionsEvent.getEntityIdScope().getId();
+ final ApplicationScope applicationScope = deIndexOldVersionsEvent.getEntityIdScope().getApplicationScope();
+ final Id entityId = deIndexOldVersionsEvent.getEntityIdScope().getId();
+ final UUID markedVersion = deIndexOldVersionsEvent.getMarkedVersion();
- return eventBuilder.deIndexOlderVersions( applicationScope, entityId )
+ return eventBuilder.deIndexOldVersions( applicationScope, entityId, markedVersion )
.toBlocking().lastOrDefault(null);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
index 1f62029..4db9f4b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
@@ -21,12 +21,12 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.util.List;
+import java.util.UUID;
import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.index.impl.IndexOperation;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -78,9 +78,11 @@ public interface EventBuilder {
* Find all versions of the entity older than the latest and de-index them.
* @param applicationScope
* @param entityId
+ * @param markedVersion
* @return
*/
- Observable<IndexOperationMessage> deIndexOlderVersions(ApplicationScope applicationScope, Id entityId );
+ Observable<IndexOperationMessage> deIndexOldVersions( ApplicationScope applicationScope,
+ Id entityId, UUID markedVersion );
/**
* A bean to hold both our observables so the caller can choose the subscription mechanism. Note that
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 5c827c6..02a7588 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -20,11 +20,10 @@
package org.apache.usergrid.corepersistence.asyncevents;
-import java.util.Collections;
+import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,7 +117,7 @@ public class EventBuilderImpl implements EventBuilder {
//TODO USERGRID-1123: Implement so we don't iterate logs twice (latest DELETED version, then to get all DELETED)
- MvccLogEntry mostRecentlyMarked = ecm.getVersions( entityId ).toBlocking()
+ MvccLogEntry mostRecentlyMarked = ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() ).toBlocking()
.firstOrDefault( null, mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
// De-indexing and entity deletes don't check log entries. We must do that first. If no DELETED logs, then
@@ -127,13 +126,16 @@ public class EventBuilderImpl implements EventBuilder {
Observable<List<MvccLogEntry>> ecmDeleteObservable = Observable.empty();
if(mostRecentlyMarked != null){
+
+ // fetch entity versions to be de-index by looking in cassandra
deIndexObservable =
- indexService.deleteEntityIndexes( applicationScope, entityId, mostRecentlyMarked.getVersion() );
+ indexService.deIndexEntity(applicationScope, entityId, mostRecentlyMarked.getVersion(),
+ getVersionsOlderThanMarked(ecm, entityId, mostRecentlyMarked.getVersion()));
ecmDeleteObservable =
- ecm.getVersions( entityId )
+ ecm.getVersionsFromMaxToMin( entityId, mostRecentlyMarked.getVersion() )
.filter( mvccLogEntry->
- UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) <= 0)
+ mvccLogEntry.getVersion().timestamp() <= mostRecentlyMarked.getVersion().timestamp() )
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
}
@@ -173,7 +175,8 @@ public class EventBuilderImpl implements EventBuilder {
@Override
- public Observable<IndexOperationMessage> deIndexOlderVersions(final ApplicationScope applicationScope, Id entityId ){
+ public Observable<IndexOperationMessage> deIndexOldVersions( final ApplicationScope applicationScope,
+ final Id entityId, final UUID markedVersion ){
if (logger.isDebugEnabled()) {
logger.debug("Removing old versions of entity {} from index in app scope {}", entityId, applicationScope );
@@ -181,24 +184,31 @@ public class EventBuilderImpl implements EventBuilder {
final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
- // find all versions of the entity that come before the provided entityId
- VersionSet latestVersions = ecm.getLatestVersion(Collections.singletonList(entityId) ).toBlocking()
- .firstOrDefault( null );
- // If there are no versions before this, allow it to return an empty observable
- Observable<IndexOperationMessage> deIndexObservable = Observable.empty();
+ return indexService.deIndexOldVersions( applicationScope, entityId,
+ getVersionsOlderThanMarked(ecm, entityId, markedVersion), markedVersion);
- if(latestVersions.getMaxVersion(entityId) != null){
+ }
- UUID latestVersion = latestVersions.getMaxVersion(entityId).getVersion();
- deIndexObservable =
- indexService.deleteEntityIndexes( applicationScope, entityId, latestVersion);
+ private List<UUID> getVersionsOlderThanMarked( final EntityCollectionManager ecm,
+ final Id entityId, final UUID markedVersion ){
- }
+ final List<UUID> versions = new ArrayList<>();
+
+ // only take last 5 versions to avoid eating memory. a tool can be built for massive cleanups for old usergrid
+ // clusters that do not have this in-line cleanup
+ ecm.getVersionsFromMaxToMin( entityId, markedVersion)
+ .take(5)
+ .forEach( mvccLogEntry -> {
+ if ( mvccLogEntry.getVersion().timestamp() < markedVersion.timestamp() ) {
+ versions.add(mvccLogEntry.getVersion());
+ }
+
+ });
- return deIndexObservable;
+ return versions;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/DeIndexOldVersionsEvent.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/DeIndexOldVersionsEvent.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/DeIndexOldVersionsEvent.java
index 59694d5..1f00e14 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/DeIndexOldVersionsEvent.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/DeIndexOldVersionsEvent.java
@@ -21,6 +21,8 @@ package org.apache.usergrid.corepersistence.asyncevents.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+import java.util.UUID;
+
/**
* An index event de-indexing documents for Entity versions older than the provided Entity
@@ -31,12 +33,16 @@ public final class DeIndexOldVersionsEvent extends AsyncEvent {
@JsonProperty
protected EntityIdScope entityIdScope;
+ @JsonProperty
+ protected UUID markedVersion;
+
public DeIndexOldVersionsEvent() {
}
- public DeIndexOldVersionsEvent(String sourceRegion, EntityIdScope entityIdScope) {
+ public DeIndexOldVersionsEvent(String sourceRegion, EntityIdScope entityIdScope, UUID markedVersion) {
super(sourceRegion);
this.entityIdScope = entityIdScope;
+ this.markedVersion = markedVersion;
}
@@ -47,4 +53,8 @@ public final class DeIndexOldVersionsEvent extends AsyncEvent {
public EntityIdScope getEntityIdScope() {
return entityIdScope;
}
+
+ public UUID getMarkedVersion() {
+ return markedVersion;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
index 54eb464..b989a9c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -20,17 +20,16 @@
package org.apache.usergrid.corepersistence.index;
+import java.util.List;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
-import rx.observables.ConnectableObservable;
/**
@@ -71,18 +70,28 @@ public interface IndexService {
Observable<IndexOperationMessage> deleteIndexEdge(final ApplicationScope applicationScope, final Edge edge);
-
-
/**
- * Delete all indexes with the specified entityId
+ * De-index all documents with the specified entityId and versions provided. This will also remove any documents
+ * where the entity is a source/target node ( index docs where this entityId is a part of connections).
*
* @param applicationScope
* @param entityId
+ * @param markedVersion
* @return
*/
- Observable<IndexOperationMessage> deleteEntityIndexes(final ApplicationScope applicationScope, final Id entityId,
- final UUID markedVersion);
+ Observable<IndexOperationMessage> deIndexEntity(final ApplicationScope applicationScope, final Id entityId,
+ final UUID markedVersion, final List<UUID> allVersionsBeforeMarked);
+ /**
+ * De-index all documents with the specified entityId and versions of the entityId provided
+ *
+ * @param applicationScope
+ * @param entityId
+ * @param markedVersion
+ * @return
+ */
+ Observable<IndexOperationMessage> deIndexOldVersions(final ApplicationScope applicationScope, final Id entityId,
+ final List<UUID> versions, UUID markedVersion);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 9509626..54b18bb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -20,12 +20,7 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,8 +51,6 @@ import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.utils.InflectionUtils;
-import org.apache.usergrid.utils.JsonUtils;
-import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Timer;
import com.google.common.base.Optional;
@@ -66,9 +59,7 @@ import com.google.inject.Singleton;
import rx.Observable;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createSearchEdgeFromSource;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromTarget;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.*;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
@@ -278,42 +269,61 @@ public class IndexServiceImpl implements IndexService {
return ObservableTimer.time( batches, addTimer );
}
- //This should look up the entityId and delete any documents with a timestamp that comes before
- //The edges that are connected will be compacted away from the graph.
+
@Override
- public Observable<IndexOperationMessage> deleteEntityIndexes( final ApplicationScope applicationScope,
- final Id entityId, final UUID markedVersion ) {
+ public Observable<IndexOperationMessage> deIndexEntity( final ApplicationScope applicationScope, final Id entityId,
+ final UUID markedVersion,
+ final List<UUID> allVersionsBeforeMarked ) {
- //bootstrap the lower modules from their caches
- final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
+ final EntityIndex ei = entityIndexFactory.
+ createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
- CandidateResults crs = ei.getAllEntityVersionsBeforeMarkedVersion( entityId, markedVersion );
- //If we get no search results, its possible that something was already deleted or
- //that it wasn't indexed yet. In either case we can't delete anything and return an empty observable..
- if(crs.isEmpty()) {
- return Observable.empty();
- }
+ final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
+ CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
+ entityId.getUuid().timestamp() ) );
+
+
+ final EntityIndexBatch batch = ei.createBatch();
+
+ // de-index each version of the entity before the marked version
+ allVersionsBeforeMarked.forEach(version -> batch.deindex(searchEdgeFromSource, entityId, version));
- UUID timeUUID = UUIDUtils.isTimeBased(entityId.getUuid()) ? entityId.getUuid() : UUIDUtils.newTimeUUID();
- //not actually sure about the timestamp but ah well. works.
- SearchEdge searchEdge = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
+
+ // for now, query the index to remove docs where the entity is source/target node and older than markedVersion
+ // TODO: investigate getting this information from graph
+ CandidateResults candidateResults = ei.getNodeDocsOlderThanMarked(entityId, markedVersion );
+ candidateResults.forEach(candidateResult -> batch.deindex(candidateResult));
+
+ return Observable.just(batch.build());
+
+ }
+
+ @Override
+ public Observable<IndexOperationMessage> deIndexOldVersions(final ApplicationScope applicationScope,
+ final Id entityId,
+ final List<UUID> versions,
+ UUID markedVersion) {
+
+ final EntityIndex ei = entityIndexFactory.
+ createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
+
+
+ final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
- timeUUID.timestamp() ) );
+ entityId.getUuid().timestamp() ) );
- final Observable<IndexOperationMessage> batches = Observable.from( crs )
- //collect results into a single batch
- .collect( () -> ei.createBatch(), ( batch, candidateResult ) -> {
- if (logger.isDebugEnabled()) {
- logger.debug("Deindexing on edge {} for entity {} added to batch", searchEdge, entityId);
- }
- batch.deindex( candidateResult );
- } )
- //return the future from the batch execution
- .map( batch ->batch.build() );
+ final EntityIndexBatch batch = ei.createBatch();
+
+ versions.forEach( version -> {
+
+ batch.deindex(searchEdgeFromSource, entityId, version);
+
+ });
+
+ return Observable.just(batch.build());
- return ObservableTimer.time(batches, indexTimer);
}
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 9de8f41..22fbb5f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.collection;
import java.util.Collection;
+import java.util.UUID;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -98,13 +99,20 @@ public interface EntityCollectionManager {
Observable<EntitySet> load( Collection<Id> entityIds );
/**
- * Get all versions of the log entry, from Max to min
+ * Get all versions of the log entry, from min to max
* @param entityId
* @return An observable stream of mvccLog entries
*/
Observable<MvccLogEntry> getVersions(final Id entityId);
/**
+ * Get all versions of the log entry, from max to min
+ * @param entityId
+ * @return An observable stream of mvccLog entries
+ */
+ Observable<MvccLogEntry> getVersionsFromMaxToMin(final Id entityId, final UUID startVersion);
+
+ /**
* Delete these versions from cassandra. Must be atomic so that read log entries are only removed. Entity data
* and log entry will be deleted
* @param entries
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index e71e6bb..70b06ba 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.UUID;
import com.netflix.astyanax.model.ConsistencyLevel;
+import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -279,6 +280,19 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
} );
}
+ @Override
+ public Observable<MvccLogEntry> getVersionsFromMaxToMin( final Id entityId, final UUID startVersion ) {
+ ValidationUtils.verifyIdentity( entityId );
+
+ return Observable.create( new ObservableIterator<MvccLogEntry>( "Log entry iterator" ) {
+ @Override
+ protected Iterator<MvccLogEntry> getIterator() {
+ return new LogEntryIterator( mvccLogEntrySerializationStrategy, applicationScope, entityId, startVersion,
+ serializationFig.getBufferSize() );
+ }
+ } );
+ }
+
@Override
public Observable<MvccLogEntry> delete( final Collection<MvccLogEntry> entries ) {
@@ -359,6 +373,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
if ( entity == null || !entity.getEntity().isPresent() ) {
final MutationBatch valueDelete =
uniqueValueSerializationStrategy.delete( applicationScope, expectedUnique );
+
deleteBatch.mergeShallow( valueDelete );
continue;
}
@@ -370,10 +385,23 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
response.addEntity( expectedUnique.getField(), entity );
}
- //TODO: explore making this an Async process
- //We'll repair it again if we have to
deleteBatch.execute();
+ // optionally sleep after read repair as some tasks immediately try to write after the delete
+ if ( serializationFig.getReadRepairDelay() > 0 ){
+
+ try {
+
+ Thread.sleep(Math.min(serializationFig.getReadRepairDelay(), 200L));
+
+ } catch (InterruptedException e) {
+
+ // do nothing if sleep fails; log and continue on
+ logger.warn("Sleep during unique value read repair failed.");
+ }
+
+ }
+
return response;
}
catch ( ConnectionException e ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 96759ba..12033fe 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -57,4 +57,9 @@ public interface SerializationFig extends GuicyFig {
@Key ( "usergrid.uniqueverify.poolsize" )
@Default( "150" )
int getUniqueVerifyPoolSize();
+
+
+ @Key ( "collection.readrepair.delay" )
+ @Default( "0" ) // in milliseconds
+ int getReadRepairDelay();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
new file mode 100644
index 0000000..de6b2bc
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
@@ -0,0 +1,128 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;
+
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Preconditions;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+
+/**
+ * Iterator that will iterate all versions of the entity from max to min
+ */
+public class LogEntryIterator implements Iterator<MvccLogEntry> {
+
+
+ private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
+ private final ApplicationScope scope;
+ private final Id entityId;
+ private final int pageSize;
+
+
+ private Iterator<MvccLogEntry> elementItr;
+ private UUID nextStart;
+ private UUID startVersion;
+
+
+ /**
+ * @param logEntrySerializationStrategy The serialization strategy to get the log entries
+ * @param scope The scope of the entity
+ * @param entityId The id of the entity
+ * @param pageSize The fetch size to get when querying the serialization strategy
+ */
+ public LogEntryIterator( final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
+ final ApplicationScope scope, final Id entityId,
+ final UUID startVersion, final int pageSize ) {
+
+ Preconditions.checkArgument( pageSize > 0, "pageSize must be > 0" );
+
+ this.logEntrySerializationStrategy = logEntrySerializationStrategy;
+ this.scope = scope;
+ this.entityId = entityId;
+ this.pageSize = pageSize;
+ this.startVersion = startVersion;
+
+ }
+
+
+ @Override
+ public boolean hasNext() {
+ if ( elementItr == null || !elementItr.hasNext() && nextStart != null ) {
+ try {
+ advance();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to query cassandra", e );
+ }
+ }
+
+ return elementItr.hasNext();
+ }
+
+
+ @Override
+ public MvccLogEntry next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more elements exist" );
+ }
+
+ return elementItr.next();
+ }
+
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException( "Remove is unsupported" );
+ }
+
+
+ /**
+ * Advance our iterator
+ */
+ public void advance() throws ConnectionException {
+
+ final int requestedSize;
+ UUID start;
+
+ if ( nextStart != null ) {
+ requestedSize = pageSize + 1;
+ start = nextStart;
+ }
+ else {
+ requestedSize = pageSize;
+ start = startVersion;
+ }
+
+ //loop through even entry that's < this one and remove it
+ List<MvccLogEntry> results = logEntrySerializationStrategy.load( scope, entityId, start, requestedSize );
+
+ //we always remove the first version if it's equal since it's returned
+ if ( nextStart != null && results.size() > 0 && results.get( 0 ).getVersion().equals( nextStart ) ) {
+ results.remove( 0 );
+ }
+
+
+
+ //we have results, set our next start. If we miss our start version (due to deletion) and we request a +1, we want to ensure we set our next, hence the >=
+ if ( results.size() >= pageSize ) {
+ nextStart = results.get( results.size() - 1 ).getVersion();
+ }
+ //nothing left to do
+ else {
+ nextStart = null;
+ }
+
+
+
+
+ elementItr = results.iterator();
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 81f900c..8ab2d41 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -113,14 +113,13 @@ public interface EntityIndex extends CPManager {
CandidateResults getAllEdgeDocuments(final IndexEdge edge, final Id entityId);
/**
- * Returns all entity documents that match the entityId and come before the marked version
+ * Returns all entity docs that match the entityId being the nodeId ( aka connections where entityId = sourceNode)
*
* @param entityId The entityId to match when searching
* @param markedVersion The version that has been marked for deletion. All version before this one must be deleted.
* @return
*/
- CandidateResults getAllEntityVersionsBeforeMarkedVersion(final Id entityId, final UUID markedVersion);
-
+ CandidateResults getNodeDocsOlderThanMarked(final Id entityId, final UUID markedVersion);
/**
* delete all application records
*
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index e81219a..3dbf5e5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -202,7 +202,7 @@ public interface IndexFig extends GuicyFig {
@Key( "elasticsearch_queue_error_sleep_ms" )
long getSleepTimeForQueueError();
- @Default("1000")
+ @Default("100")
@Key( ELASTICSEARCH_VERSION_QUERY_LIMIT )
int getVersionQueryLimit();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 10ee91e..3b60b57 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -533,7 +533,9 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
@Override
- public CandidateResults getAllEntityVersionsBeforeMarkedVersion( final Id entityId, final UUID markedVersion ) {
+ public CandidateResults getNodeDocsOlderThanMarked(final Id entityId, final UUID markedVersion ) {
+
+ // TODO: investigate if functionality via iterator so a caller can page the deletion until all is gone
Preconditions.checkNotNull( entityId, "entityId cannot be null" );
Preconditions.checkNotNull(markedVersion, "markedVersion cannot be null");
@@ -544,12 +546,8 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final long markedTimestamp = markedVersion.timestamp();
- // never let the limit be less than 2 as there are potential indefinite paging issues
- final int searchLimit = Math.max(2, indexFig.getVersionQueryLimit());
-
- // this query will find the document for the entity itself
- final QueryBuilder entityQuery = QueryBuilders
- .termQuery(IndexingUtils.ENTITY_ID_FIELDNAME, IndexingUtils.entityId(entityId));
+ // never let this fetch more than 100 to save memory
+ final int searchLimit = Math.min(100, indexFig.getVersionQueryLimit());
// this query will find all the documents where this entity is a source/target node
final QueryBuilder nodeQuery = QueryBuilders
@@ -562,49 +560,25 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
long queryTimestamp = 0L;
- while(true){
+ QueryBuilder timestampQuery = QueryBuilders
+ .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
+ .gte(queryTimestamp)
+ .lt(markedTimestamp);
- QueryBuilder timestampQuery = QueryBuilders
- .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
- .gte(queryTimestamp)
- .lte(markedTimestamp);
+ QueryBuilder finalQuery = QueryBuilders
+ .boolQuery()
+ .must(timestampQuery)
+ .must(nodeQuery);
- QueryBuilder entityQueryWithTimestamp = QueryBuilders
- .boolQuery()
- .must(entityQuery)
- .must(timestampQuery);
+ searchResponse = srb
+ .setQuery(finalQuery)
+ .setSize(searchLimit)
+ .execute()
+ .actionGet();
- QueryBuilder finalQuery = QueryBuilders
- .boolQuery()
- .should(entityQueryWithTimestamp)
- .should(nodeQuery)
- .minimumNumberShouldMatch(1);
- searchResponse = srb
- .setQuery(finalQuery)
- .setSize(searchLimit)
- .execute()
- .actionGet();
+ candidates = aggregateScrollResults(candidates, searchResponse, markedVersion);
- int responseSize = searchResponse.getHits().getHits().length;
- if(responseSize == 0){
- break;
- }
-
- candidates = aggregateScrollResults(candidates, searchResponse, markedVersion);
-
- // update queryTimestamp to be the timestamp of the last entity returned from the query
- queryTimestamp = (long) searchResponse
- .getHits().getAt(responseSize - 1)
- .getSource().get(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME);
-
-
- if(responseSize < searchLimit){
-
- break;
- }
-
- }
}
catch ( Throwable t ) {
logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7143cbaf/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 008ec80..c84635d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -383,47 +383,6 @@ public class EntityIndexTest extends BaseIT {
}
- /**
- * Tests that we aggregate results only before the halfway version point.
- */
- @Test
- public void testScollingDeindex() {
-
- int numberOfEntities = 1000;
- int versionToSearchFor = numberOfEntities / 2;
-
-
- UUID entityUUID = UUID.randomUUID();
- Id entityId = new SimpleId( "mehCar" );
-
- Map entityMap = new HashMap() {{
- put( "name", "Toyota Corolla" );
- put( "introduced", 1966 );
- put( "topspeed", 111 );
- }};
-
- Entity[] entity = new Entity[numberOfEntities];
- for(int i = 0; i < numberOfEntities; i++) {
- entity[i] = EntityIndexMapUtils.fromMap( entityMap );
- EntityUtils.setId(entity[i], entityId);
- EntityUtils.setVersion(entity[i], UUIDGenerator.newTimeUUID());
- entity[i].setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID));
-
- IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, System.currentTimeMillis()*1000 );
-
- //index the new entity. This is where the loop will be set to create like 100 entities.
- indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i] ).build()).subscribe();
-
- }
- entityIndex.refreshAsync().toBlocking().first();
-
- CandidateResults candidateResults = entityIndex
- .getAllEntityVersionsBeforeMarkedVersion( entity[versionToSearchFor].getId(),
- entity[versionToSearchFor].getVersion() );
- assertEquals( 501, candidateResults.size() );
- }
-
-
private CandidateResults testQuery( final SearchEdge scope, final SearchTypes searchTypes,
final String queryString,
[38/44] usergrid git commit: Separate create vs. get column family
logic.
Posted by sn...@apache.org.
Separate create vs. get column family logic.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cc7a8678
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cc7a8678
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cc7a8678
Branch: refs/heads/usergrid-1268-akka-211
Commit: cc7a8678481dccda00d867fc363abc0d404f0c37
Parents: fcb347c
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Jun 6 12:23:46 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Jun 6 12:23:46 2016 -0400
----------------------------------------------------------------------
.../cassandra/AstyanaxLockManagerImpl.java | 87 ++++++++++++--------
1 file changed, 53 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc7a8678/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 90b9d57..e26aed4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -62,13 +62,13 @@ public class AstyanaxLockManagerImpl implements LockManager {
this.cassandraFig = cassandraFig;
this.cassandraCluster = cassandraCluster;
+ this.keyspace = cassandraCluster.getLocksKeyspace();
}
@Override
public void setup() {
try {
- keyspace = cassandraCluster.getLocksKeyspace();
createLocksKeyspace();
columnFamily = createLocksColumnFamily();
} catch (ConnectionException e) {
@@ -78,7 +78,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
@Override
- public Lock createLock(final UUID applicationId, final String... path ){
+ public Lock createLock(final UUID applicationId, final String... path ) {
String lockPath = LockPathBuilder.buildPath( applicationId, path );
@@ -114,7 +114,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
ColumnPrefixDistributedRowLock<String> lock =
- new ColumnPrefixDistributedRowLock<>(keyspace, columnFamily, lockPath)
+ new ColumnPrefixDistributedRowLock<>(keyspace, getLocksColumnFamily(), lockPath)
.expireLockAfter( lockExpiration, TimeUnit.MILLISECONDS)
.withConsistencyLevel(consistencyLevel);
@@ -124,56 +124,75 @@ public class AstyanaxLockManagerImpl implements LockManager {
}
+ private ColumnFamily getLocksColumnFamily() {
- private ColumnFamily createLocksColumnFamily() throws ConnectionException {
+ if ( columnFamily == null ) {
- ColumnFamily<String, String> CF_LOCKS = ColumnFamily.newColumnFamily(
- CF_NAME, StringSerializer.get(), StringSerializer.get() );
+ columnFamily = ColumnFamily.newColumnFamily(
+ CF_NAME, StringSerializer.get(), StringSerializer.get() );
- final KeyspaceDefinition keyspaceDefinition = keyspace.describeKeyspace();
- final ColumnFamilyDefinition existing =
- keyspaceDefinition.getColumnFamily( CF_LOCKS.getName() );
+ if ( logger.isDebugEnabled() ) {
+ try {
+ final KeyspaceDefinition kd = keyspace.describeKeyspace();
+ final ColumnFamilyDefinition cfd = kd.getColumnFamily( columnFamily.getName() );
+ Map<String, Object> options = new HashMap<>( 1 );
+ options.put( "gc_grace_seconds", cfd.getGcGraceSeconds() );
+ options.put( "caching", cfd.getCaching() );
+ options.put( "compaction_strategy", cfd.getCompactionStrategy() );
+ options.put( "compaction_strategy_options", cfd.getCompactionStrategyOptions() );
+ logger.debug( "Locks column family {} exists with options: {}", cfd.getName(), options);
+
+ } catch ( ConnectionException ce ) {
+ logger.warn("Error connecting to Cassandra for debug column family info", ce);
+ }
+ }
+ }
- if ( existing != null ) {
+ return columnFamily;
+ }
- Map<String, Object> existingOptions = new HashMap<>(1);
- existingOptions.put("gc_grace_seconds", existing.getGcGraceSeconds());
- existingOptions.put("caching", existing.getCaching());
- existingOptions.put("compaction_strategy", existing.getCompactionStrategy());
- existingOptions.put("compaction_strategy_options", existing.getCompactionStrategyOptions());
- logger.info( "Locks column family {} exists with options: {}", existing.getName(),
- existingOptions.toString() );
+ private ColumnFamily createLocksColumnFamily() throws ConnectionException {
- return CF_LOCKS;
- }
+ ColumnFamily<String, String> cflocks = ColumnFamily.newColumnFamily(
+ CF_NAME, StringSerializer.get(), StringSerializer.get() );
- MultiTenantColumnFamilyDefinition columnFamilyDefinition = new MultiTenantColumnFamilyDefinition(
- CF_LOCKS,
- BytesType.class.getSimpleName(),
- UTF8Type.class.getSimpleName(),
- BytesType.class.getSimpleName(),
- MultiTenantColumnFamilyDefinition.CacheOption.ALL
- );
+ final KeyspaceDefinition kd = keyspace.describeKeyspace();
+ final ColumnFamilyDefinition cfdef = kd.getColumnFamily( cflocks.getName() );
- Map<String, Object> cfOptions = columnFamilyDefinition.getOptions();
+ if ( cfdef == null ) {
- // Additionally set the gc grace low
- cfOptions.put("gc_grace_seconds", 60);
+ // create only if does not already exist
+ MultiTenantColumnFamilyDefinition mtcfd = new MultiTenantColumnFamilyDefinition(
+ cflocks,
+ BytesType.class.getSimpleName(),
+ UTF8Type.class.getSimpleName(),
+ BytesType.class.getSimpleName(),
+ MultiTenantColumnFamilyDefinition.CacheOption.ALL
+ );
- keyspace.createColumnFamily( columnFamilyDefinition.getColumnFamily() , cfOptions );
+ Map<String, Object> cfOptions = mtcfd.getOptions();
- logger.info( "Created column family {}", columnFamilyDefinition.getOptions() );
+ // Additionally set the gc grace low
+ cfOptions.put( "gc_grace_seconds", 60 );
- return columnFamilyDefinition.getColumnFamily();
- }
+ keyspace.createColumnFamily( mtcfd.getColumnFamily(), cfOptions );
+ logger.info( "Created column family {}", mtcfd.getOptions() );
- private void createLocksKeyspace() throws ConnectionException {
+ cflocks = mtcfd.getColumnFamily();
+
+ } else {
+ return getLocksColumnFamily();
+ }
+
+ return cflocks;
+ }
+ private void createLocksKeyspace() throws ConnectionException {
ImmutableMap.Builder<String, Object> strategyOptions = getKeySpaceProps();
[34/44] usergrid git commit: Quick fix for Service tests: ensure
lockManager gets initialized.
Posted by sn...@apache.org.
Quick fix for Service tests: ensure lockManager gets initialized.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/66bb5cd4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/66bb5cd4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/66bb5cd4
Branch: refs/heads/usergrid-1268-akka-211
Commit: 66bb5cd4911e10c308dae05d046033ba4d89acdc
Parents: 17809ac
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 3 12:08:31 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 3 12:08:31 2016 -0400
----------------------------------------------------------------------
stack/services/src/test/resources/usergrid-test-context.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/66bb5cd4/stack/services/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-test-context.xml b/stack/services/src/test/resources/usergrid-test-context.xml
index ec1e2d2..8b31421 100644
--- a/stack/services/src/test/resources/usergrid-test-context.xml
+++ b/stack/services/src/test/resources/usergrid-test-context.xml
@@ -49,6 +49,7 @@
<bean id="coreManager" class="org.apache.usergrid.persistence.CoreSchemaManager">
<constructor-arg ref="setup"/>
<constructor-arg ref="cassandraCluster"/>
+ <constructor-arg ref="injector"/>
</bean>
<bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
[23/44] usergrid git commit: Made max-retries and intervals
configurable for both lock manager and entity manager factory,
also better error logging.
Posted by sn...@apache.org.
Made max-retries and intervals configurable for both lock manager and entity manager factory, also better error logging.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/61a35a04
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/61a35a04
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/61a35a04
Branch: refs/heads/usergrid-1268-akka-211
Commit: 61a35a041598af0e336199343ed6a489454980f6
Parents: fcd00e8
Author: Dave Johnson <sn...@apache.org>
Authored: Thu May 26 11:10:33 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu May 26 11:10:33 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 83 ++++++++++++++++----
.../cassandra/AstyanaxLockManagerImpl.java | 17 +++-
.../exception/CollectionRuntimeException.java | 11 +++
3 files changed, 93 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b6fbc2a..ee28765 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -20,11 +20,10 @@ import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.UncheckedExecutionException;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
-import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
@@ -39,6 +38,7 @@ import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
@@ -68,6 +68,7 @@ import java.util.*;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import static org.apache.usergrid.persistence.Schema.initLock;
/**
@@ -107,7 +108,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final GraphManagerFactory graphManagerFactory;
private final IndexSchemaCacheFactory indexSchemaCacheFactory;
- public static final String MANAGEMENT_APP_MAX_RETRIES= "management.app.max.retries";
+ public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries";
+ public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval";
public CpEntityManagerFactory(
@@ -177,12 +179,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
}
- if ( app == null && throwable != null && throwable.getCause() instanceof BadRequestException) {
- // probably means schema has not been created yet
+ final boolean missingKeyspace;
+ if ( throwable instanceof CollectionRuntimeException ) {
+ CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
+ missingKeyspace = cre.isMissingKeyspace();
} else {
- throw new RuntimeException( "Error getting application " + appId, throwable );
+ missingKeyspace = false;
}
+ if ( app == null && !missingKeyspace ) {
+ throw new RuntimeException( "Error getting application " + appId, throwable );
+
+ } // else keyspace is missing because setup/bootstrap not done yet
+
return entityManager;
}
});
@@ -190,34 +199,77 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private void checkManagementApp(Properties properties) {
+
int maxRetries = 100;
try {
- maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_MAX_RETRIES, "100" ));
+ maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_MAXRETRIES, "100" ));
+
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + MANAGEMENT_APP_INIT_MAXRETRIES + ". Will use " + maxRetries, e );
+ }
+
+ int interval = 1000;
+ try {
+ interval = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_INTERVAL, "1000" ));
} catch ( Exception e ) {
- logger.error("Error parsing " + MANAGEMENT_APP_MAX_RETRIES + ". Will use " + maxRetries, e );
+ logger.error("Error parsing " + MANAGEMENT_APP_INIT_INTERVAL + ". Will use " + maxRetries, e );
}
// hold up construction until we can access the management app
int retries = 0;
boolean managementAppFound = false;
+ boolean bootstrapping = false;
+ Set<Class> seenBefore = new HashSet<>(10);
while ( !managementAppFound && retries++ < maxRetries ) {
try {
- getEntityManager( getManagementAppId() ).getApplication();
+ // bypass entity manager cache and get managementApp
+ managementApp = _getEntityManager( getManagementAppId() ).getApplication();
managementAppFound = true;
} catch ( Throwable t ) {
- String msg = "Error " + t.getClass() + " getting management app on try " + retries;
- if ( logger.isDebugEnabled() ) {
+
+ if ( t instanceof CollectionRuntimeException ) {
+ CollectionRuntimeException cre = (CollectionRuntimeException)t;
+ if ( cre.isMissingKeyspace() ) {
+ // we're bootstrapping, ignore this and continue
+ bootstrapping = true;
+ break;
+ }
+ }
+ Throwable cause = t;
+
+ // there was an error, be as informative as possible
+ StringBuilder sb = new StringBuilder();
+ sb.append(retries).append(": Error (");
+
+ if ( t instanceof UncheckedExecutionException ) {
+ UncheckedExecutionException uee = (UncheckedExecutionException)t;
+ if ( uee.getCause() instanceof RuntimeException ) {
+ cause = uee.getCause().getCause();
+ sb.append(cause.getClass().getSimpleName()).append(") ")
+ .append(uee.getCause().getMessage());
+ } else {
+ cause = uee.getCause();
+ sb.append(cause.getClass().getSimpleName()).append(") ").append(t.getMessage());
+ }
+ } else {
+ sb.append(t.getCause().getClass().getSimpleName()).append(") ").append(t.getMessage());
+ }
+
+ String msg = sb.toString();
+ if ( !seenBefore.contains( cause.getClass() ) ) {
logger.error( msg, t);
} else {
logger.error(msg);
}
- try { Thread.sleep( 1000 ); } catch (InterruptedException ignored) {}
+ seenBefore.add( cause.getClass() );
+
+ try { Thread.sleep( interval ); } catch (InterruptedException ignored) {}
}
}
- if ( !managementAppFound ) {
+ if ( !managementAppFound && !bootstrapping ) {
// exception here will prevent WAR from being deployed
throw new RuntimeException( "Unable to get management app after " + retries + " retries" );
}
@@ -323,8 +375,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// Ensure the management application is initialized
initMgmtAppInternal();
- EntityManager managementEm = getEntityManager( getManagementAppId() );
- EntityManager appEm = getEntityManager(applicationId);
+ // Get entity managers by bypassing the entity manager cache because it expects apps to already exist
+ final EntityManager managementEm = _getEntityManager( getManagementAppId() );
+ EntityManager appEm = _getEntityManager(applicationId);
final String appName = buildAppName(organizationName, name);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 9bf6694..6acce47 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -22,6 +22,8 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.connectionpool.exceptions.NoAvailableHostsException;
+import com.netflix.astyanax.connectionpool.exceptions.PoolTimeoutException;
import com.netflix.astyanax.ddl.ColumnFamilyDefinition;
import com.netflix.astyanax.ddl.KeyspaceDefinition;
import com.netflix.astyanax.model.ColumnFamily;
@@ -63,6 +65,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
int maxRetries = cassandraFig.getLockManagerInitRetries();
int retries = 0;
boolean famReady = false;
+ Set<Class> seenBefore = new HashSet<>(10);
while ( !famReady && retries++ < maxRetries ) {
try {
keyspace = cassandraCluster.getLocksKeyspace();
@@ -71,13 +74,21 @@ public class AstyanaxLockManagerImpl implements LockManager {
famReady = true;
} catch ( Throwable t ) {
- String msg = "Error " + t.getClass().getSimpleName() + " creating locks keyspace try " + retries;
- if ( logger.isDebugEnabled() ) {
+ final String msg;
+ if ( t instanceof PoolTimeoutException || t instanceof NoAvailableHostsException) {
+ msg = retries + ": Cannot connect to Cassandra (" + t.getClass().getSimpleName() + ")";
+ } else {
+ msg = retries + ": Error (" + t.getClass().getSimpleName() + ") tries=" + retries;
+ }
+ if ( !seenBefore.contains( t.getClass() ) ) {
logger.error( msg, t );
} else {
logger.error( msg );
}
- try { Thread.sleep( cassandraFig.getLockManagerInitInterval() ); } catch (InterruptedException ignored) {}
+ seenBefore.add( t.getClass() );
+ try {
+ Thread.sleep( cassandraFig.getLockManagerInitInterval() );
+ } catch (InterruptedException ignored) {}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
index 431122e..f27e2d7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
@@ -17,6 +17,7 @@
*/
package org.apache.usergrid.persistence.collection.exception;
+import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -56,6 +57,16 @@ public class CollectionRuntimeException extends RuntimeException {
this.applicationScope = scope;
}
+ public boolean isMissingKeyspace() {
+ if ( getCause() instanceof BadRequestException ) {
+ BadRequestException bre = (BadRequestException)getCause();
+ String msg = bre.getMessage();
+ if ( msg.contains("Keyspace") && msg.contains( "does not exist" ) ) {
+ return true;
+ }
+ }
+ return false;
+ }
public ApplicationScope getApplicationScope() {
return applicationScope;
[05/44] usergrid git commit: Only allow GET access to users/me
Posted by sn...@apache.org.
Only allow GET access to users/me
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1d0e73b3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1d0e73b3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1d0e73b3
Branch: refs/heads/usergrid-1268-akka-211
Commit: 1d0e73b3f15d634a484ec3e425f944317aefa1b3
Parents: cd363f4
Author: Dave Johnson <sn...@apache.org>
Authored: Tue May 10 17:41:40 2016 -0700
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue May 10 17:41:40 2016 -0700
----------------------------------------------------------------------
.../usergrid/rest/security/SecuredResourceFilterFactory.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d0e73b3/stack/rest/src/main/java/org/apache/usergrid/rest/security/SecuredResourceFilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/security/SecuredResourceFilterFactory.java b/stack/rest/src/main/java/org/apache/usergrid/rest/security/SecuredResourceFilterFactory.java
index 67cf248..bd1ab46 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/security/SecuredResourceFilterFactory.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/security/SecuredResourceFilterFactory.java
@@ -450,7 +450,7 @@ public class SecuredResourceFilterFactory implements DynamicFeature {
String path = request.getUriInfo().getPath().toLowerCase().replace(applicationName, "");
String perm = getPermissionFromPath( em.getApplicationRef().getUuid(), operation, path );
- if ( "/users/me".equals( path ) ) {
+ if ( "/users/me".equals( path ) && request.getMethod().equalsIgnoreCase( "get" )) {
// shortcut the permissions checking, the "me" end-point is always allowed
logger.debug("Allowing {} access to /users/me", getSubject().toString() );
return;
[17/44] usergrid git commit: Revert all changes to ServiceManager
Posted by sn...@apache.org.
Revert all changes to ServiceManager
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7d2de990
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7d2de990
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7d2de990
Branch: refs/heads/usergrid-1268-akka-211
Commit: 7d2de990705c0fb8380a03620862815854e9fd73
Parents: 6daba1d
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:20:24 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:20:24 2016 -0400
----------------------------------------------------------------------
.../usergrid/services/ServiceManager.java | 25 +++++++++++---------
1 file changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7d2de990/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index e5ef407..a9892f5 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@ -17,29 +17,32 @@
package org.apache.usergrid.services;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.lang.StringUtils;
+import java.lang.reflect.Modifier;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
import org.apache.usergrid.batch.service.SchedulerService;
import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.mq.QueueManager;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.services.ServiceParameter.IdParameter;
import org.apache.usergrid.services.applications.ApplicationsService;
import org.apache.usergrid.services.exceptions.UndefinedServiceEntityTypeException;
import org.apache.usergrid.utils.ListUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.ApplicationContext;
-import java.lang.reflect.Modifier;
-import java.util.*;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
import static org.apache.usergrid.utils.InflectionUtils.pluralize;
[11/44] usergrid git commit: Test for doubly nested field,
plus some formatting changes.
Posted by sn...@apache.org.
Test for doubly nested field, plus some formatting changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0b416203
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0b416203
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0b416203
Branch: refs/heads/usergrid-1268-akka-211
Commit: 0b4162035ee7d54199bd2250c2b3ac31354516b1
Parents: 3eec8e5
Author: Dave Johnson <sn...@apache.org>
Authored: Thu May 19 08:57:42 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu May 19 08:57:42 2016 -0400
----------------------------------------------------------------------
.../read/search/CandidateEntityFilter.java | 16 +++++++-----
.../apache/usergrid/persistence/IndexIT.java | 27 +++++++++++++++-----
2 files changed, 30 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0b416203/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index 261259b..e02e6c7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -52,9 +52,9 @@ import rx.Observable;
/**
- * Loads entities from an incoming CandidateResult emissions into entities, then streams them on
- * performs internal buffering for efficiency. Note that all entities may not be emitted if our load crosses page boundaries. It is up to the
- * collector to determine when to stop streaming entities.
+ * Loads entities from an incoming CandidateResult emissions into entities, then streams them on performs internal
+ * buffering for efficiency. Note that all entities may not be emitted if our load crosses page boundaries.
+ * It is up to the collector to determine when to stop streaming entities.
*/
public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate>, FilterResult<Entity>> {
@@ -93,11 +93,12 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
entityCollectionManagerFactory.createCollectionManager( applicationScope );
- final EntityIndex applicationIndex =
- entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
+ final EntityIndex applicationIndex = entityIndexFactory
+ .createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
//buffer them to get a page size we can make 1 network hop
- final Observable<FilterResult<Entity>> searchIdSetObservable = candidateResultsObservable.buffer( pipelineContext.getLimit() )
+ final Observable<FilterResult<Entity>> searchIdSetObservable =
+ candidateResultsObservable.buffer( pipelineContext.getLimit() )
//load them
.flatMap( candidateResults -> {
@@ -198,7 +199,8 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
/**
- * Our collector to collect entities. Not quite a true collector, but works within our operational flow as this state is mutable and difficult to represent functionally
+ * Our collector to collect entities. Not quite a true collector, but works within our operational
+ * flow as this state is mutable and difficult to represent functionally
*/
private static final class EntityVerifier {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0b416203/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index 3b9f95a..aaf4c33 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -477,6 +477,10 @@ public class IndexIT extends AbstractCoreIT {
put("xfactor", 5.1);
put("rando", "bar");
put("mondo", "2000");
+ put("frosting", "chocolate");
+ put("misc", new HashMap() {{
+ put("range", "open");
+ }});
}});
}};
em.create("names", entity1);
@@ -488,12 +492,17 @@ public class IndexIT extends AbstractCoreIT {
put("xfactor", 5.1);
put("rando", "bar");
put("mondo", "2001");
+ put("frosting", "orange");
+ put("misc", new HashMap() {{
+ put("range", "open");
+ }});
}});
}};
em.create("names", entity2);
app.refreshIndex();
+ // simple single-field select mapping
{
Query query = Query.fromQL("select status where status = 'pickled'");
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
@@ -502,6 +511,7 @@ public class IndexIT extends AbstractCoreIT {
assertTrue(first.getDynamicProperties().size() == 2);
}
+ // simple single-field plus nested field select mapping
{
Query query = Query.fromQL( "select status, data.rando where data.rando = 'bar'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
@@ -518,23 +528,28 @@ public class IndexIT extends AbstractCoreIT {
assertTrue( first.getDynamicProperties().size() == 3 );
}
+ // multiple nested fields with one doubly-nesed field
{
- // query for only one bogus field name should return empty entities
- Query query = Query.fromQL( "select data.rando,data.mondo where status = 'pickled'" );
+ Query query = Query.fromQL( "select data.rando, data.mondo, data.misc.range where status = 'pickled'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
Entity first = r.getEntities().get( 0 );
- assertNotNull( first.getProperty("data") );
- assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
- assertEquals( ((Map<String, Object>)first.getProperty("data")).get("mondo"), "2001" );
+ Map<String, Object> data = ((Map<String, Object>)first.getProperty("data"));
+ assertNotNull( data );
+ assertEquals( data.get("rando"), "bar" );
+ assertEquals( data.get("mondo"), "2001" );
+ assertNull( data.get("frosting") );
+
+ Map<String, Object> misc = (Map<String, Object>)data.get("misc");
+ assertEquals( misc.get("range"), "open" );
assertTrue( first.getDynamicProperties().size() == 2 );
}
+ // query for one bogus field name should return empty entities
{
- // query for only one bogus field name should return empty entities
Query query = Query.fromQL( "select data.bogusfieldname where status = 'pickled'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
[10/44] usergrid git commit: Two fields.
Posted by sn...@apache.org.
Two fields.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3eec8e55
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3eec8e55
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3eec8e55
Branch: refs/heads/usergrid-1268-akka-211
Commit: 3eec8e55eb57b889be1f2d0d13b44d20a98879dd
Parents: 596e0fc
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 18 18:58:17 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 18 18:58:17 2016 -0400
----------------------------------------------------------------------
.../src/test/java/org/apache/usergrid/persistence/IndexIT.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3eec8e55/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index 7e38f17..3b9f95a 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -476,6 +476,7 @@ public class IndexIT extends AbstractCoreIT {
put("data", new HashMap() {{
put("xfactor", 5.1);
put("rando", "bar");
+ put("mondo", "2000");
}});
}};
em.create("names", entity1);
@@ -486,6 +487,7 @@ public class IndexIT extends AbstractCoreIT {
put("data", new HashMap() {{
put("xfactor", 5.1);
put("rando", "bar");
+ put("mondo", "2001");
}});
}};
em.create("names", entity2);
@@ -518,7 +520,7 @@ public class IndexIT extends AbstractCoreIT {
{
// query for only one bogus field name should return empty entities
- Query query = Query.fromQL( "select data.rando where status = 'pickled'" );
+ Query query = Query.fromQL( "select data.rando,data.mondo where status = 'pickled'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
@@ -526,6 +528,7 @@ public class IndexIT extends AbstractCoreIT {
assertNotNull( first.getProperty("data") );
assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
+ assertEquals( ((Map<String, Object>)first.getProperty("data")).get("mondo"), "2001" );
assertTrue( first.getDynamicProperties().size() == 2 );
}
[16/44] usergrid git commit: Merge branch 'release-2.1.1' into
usergrid-1283-mgmt-app-init
Posted by sn...@apache.org.
Merge branch 'release-2.1.1' into usergrid-1283-mgmt-app-init
Conflicts:
stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6daba1d7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6daba1d7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6daba1d7
Branch: refs/heads/usergrid-1268-akka-211
Commit: 6daba1d7d3dd9ce65644229af87f0440cee53117
Parents: 8ebc28d 0970e1d
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:08:06 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:08:06 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 5 --
.../corepersistence/CpEntityManager.java | 3 -
.../corepersistence/CpEntityManagerFactory.java | 72 +++++++++-----------
.../usergrid/services/ServiceManager.java | 27 ++++----
4 files changed, 46 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6daba1d7/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6daba1d7/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 924dd10,91a936d..f92fc9e
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -141,66 -142,12 +141,79 @@@ public class CpEntityManagerFactory imp
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
- // this line always needs to be last due to the temporary circular dependency until spring is removed
-
- this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
- getManagementEntityManager() );
-
- //this line always needs to be last due to the temporary cicular dependency until spring is removed
+ int entityManagerCacheSize = 100;
+ try {
+ entityManagerCacheSize = Integer.parseInt(
+ cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e );
+ }
- this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
- getManagementEntityManager() );
+ entityManagers = CacheBuilder.newBuilder()
+ .maximumSize(entityManagerCacheSize)
+ .build(new CacheLoader<UUID, EntityManager>() {
+ public EntityManager load( UUID appId ) { // no checked exception
+
- // get entity manager and ensure it can get its own application
-
++ // create new entity manager and pre-fetch its application
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Exception exception = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Exception e) {
+ exception = e;
+ }
+
+ // the management app is a special case
+
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+
- if ( app != null && entityManager != null ) {
-
++ if ( app != null ) {
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
+
- } else if ( entityManager == null && managementAppEntityManager != null ) {
-
++ } else if ( managementAppEntityManager != null ) {
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
-
- } else {
-
- // fetch failed and we have nothing cached, we must be bootstrapping
- logger.info("managementAppEntityManager is null, bootstrapping in progress");
+ }
++ }
- } else { // not the management app, so blow up if app cannot be fetched
-
- if (app == null) {
- throw new RuntimeException( "Error getting application " + appId, exception );
- }
++ if (app == null) {
++ throw new RuntimeException( "Error getting application " + appId, exception );
+ }
+
+ return entityManager;
+ }
+ });
++
++ // hold up construction until we can access the management app
++ int maxRetries = 1000;
++ int retries = 0;
++ boolean managementAppFound = false;
++ Set<Class> seenBefore = new HashSet<>(100);
++ while ( !managementAppFound && retries++ < maxRetries ) {
++ try {
++ getEntityManager( getManagementAppId() ).getApplication();
++ managementAppFound = true;
++
++ } catch ( Throwable t ) {
++ if ( seenBefore.contains( t.getClass() )) { // don't log full stack trace if we've seen same before
++ logger.error("Error {} getting management app on try {}", t.getClass().getSimpleName(), retries);
++ } else {
++ logger.error("Error getting management app on try {}", t.getClass().getSimpleName(), t);
++ }
++ }
++ }
++
++ if ( !managementAppFound ) {
++ // exception here will prevent WAR from being deployed
++ throw new RuntimeException( "Unable to get management app after " + retries + " retries" );
++ }
++
++ // this line always needs to be last due to the temporary circular dependency until spring is removed
++ applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class)
++ .getInstance( getManagementEntityManager() );
}
@@@ -255,7 -187,7 +252,6 @@@
}
--
@Override
public EntityManager getEntityManager(UUID applicationId) {
try {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6daba1d7/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index 993bf35,a9892f5..e5ef407
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@@ -17,32 -17,32 +17,29 @@@
package org.apache.usergrid.services;
--import java.lang.reflect.Modifier;
--import java.util.*;
--import java.util.concurrent.ExecutionException;
--import java.util.concurrent.TimeUnit;
--
--import org.slf4j.Logger;
--import org.slf4j.LoggerFactory;
--import org.springframework.context.ApplicationContext;
++import com.google.common.cache.CacheBuilder;
++import com.google.common.cache.CacheLoader;
++import com.google.common.cache.LoadingCache;
++import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.batch.service.SchedulerService;
import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.mq.QueueManager;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
--import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.services.ServiceParameter.IdParameter;
import org.apache.usergrid.services.applications.ApplicationsService;
import org.apache.usergrid.services.exceptions.UndefinedServiceEntityTypeException;
import org.apache.usergrid.utils.ListUtils;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
++import org.springframework.context.ApplicationContext;
--import org.apache.commons.lang.StringUtils;
--
--import com.google.common.cache.CacheBuilder;
--import com.google.common.cache.CacheLoader;
--import com.google.common.cache.LoadingCache;
++import java.lang.reflect.Modifier;
++import java.util.*;
++import java.util.concurrent.ExecutionException;
++import java.util.concurrent.TimeUnit;
import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
import static org.apache.usergrid.utils.InflectionUtils.pluralize;
[20/44] usergrid git commit: Revert all changes to
CpEntityManagerFactory
Posted by sn...@apache.org.
Revert all changes to CpEntityManagerFactory
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ba10e7f9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ba10e7f9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ba10e7f9
Branch: refs/heads/usergrid-1268-akka-211
Commit: ba10e7f94a4b55bb0c89fc209d14a7a5687760fe
Parents: b0fba68
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:29:57 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:29:57 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 84 ++++++--------------
1 file changed, 25 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ba10e7f9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f92fc9e..d2417be 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -16,16 +16,14 @@
package org.apache.usergrid.corepersistence;
-import java.util.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
-
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
import org.apache.commons.lang.StringUtils;
-
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
@@ -34,15 +32,7 @@ import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
-import org.apache.usergrid.persistence.AbstractEntity;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
@@ -57,30 +47,23 @@ import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.MarkedEdge;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.*;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
-
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import com.google.inject.TypeLiteral;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
import rx.Observable;
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
+import java.util.*;
+import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
@@ -265,19 +248,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager(
- this,
- cassandraService,
- counterUtils,
- indexService,
- managerCache,
- metricsFactory,
- entityManagerFig,
- graphManagerFactory,
- collectionService,
- connectionService,
- indexSchemaCacheFactory,
- applicationId );
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache,
+ metricsFactory, entityManagerFig, graphManagerFactory, collectionService, connectionService,indexSchemaCacheFactory, applicationId );
+
return em;
}
@@ -289,8 +262,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public Entity createApplicationV2(
- String orgName, String name, UUID applicationId, Map<String, Object> properties, boolean forMigration)
- throws Exception {
+ String orgName, String name, UUID applicationId, Map<String, Object> properties, boolean forMigration) throws Exception {
String appName = buildAppName( orgName, name );
@@ -392,9 +364,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// find application_info for application to delete
- migrateAppInfo(
- applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS,
- CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking().lastOrDefault( null );
+ migrateAppInfo(applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS, CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking()
+ .lastOrDefault( null );
}
//TODO: return status for restore
@@ -436,13 +407,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
* @return
* @throws Exception
*/
- private Observable migrateAppInfo(
- final UUID applicationUUID, final String deleteTypeName, final String createCollectionName,
- final String createTypeName ) throws Exception {
-
- final ApplicationScope managementAppScope =
- CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
+ private Observable migrateAppInfo(final UUID applicationUUID, final String deleteTypeName, final String createCollectionName, final String createTypeName ) throws Exception {
+ final ApplicationScope managementAppScope = CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
final EntityManager managementEm = getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
//the application id we will be removing
@@ -497,8 +464,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
final Id managementAppId = CpNamingUtils.getManagementApplicationId();
final EntityIndex aei = getManagementIndex();
final GraphManager managementGraphManager = managerCache.getGraphManager(managementAppScope);
- final Edge createEdge =
- CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
+ final Edge createEdge = CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
final Observable createNodeGraph = managementGraphManager.writeEdge(createEdge);
[19/44] usergrid git commit: Revert all changes to CpEntityManager
Posted by sn...@apache.org.
Revert all changes to CpEntityManager
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b0fba682
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b0fba682
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b0fba682
Branch: refs/heads/usergrid-1268-akka-211
Commit: b0fba682d652195f56f6e49b09b488ca677780a6
Parents: e299024
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:28:05 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:28:05 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b0fba682/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 13b5d1f..68f5d71 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -243,16 +243,16 @@ public class CpEntityManager implements EntityManager {
* @param applicationId
*/
public CpEntityManager( final CassandraService cass,
- final CounterUtils counterUtils,
- final AsyncEventService indexService,
- final ManagerCache managerCache,
- final MetricsFactory metricsFactory,
- final EntityManagerFig entityManagerFig,
- final GraphManagerFactory graphManagerFactory,
- final CollectionService collectionService,
- final ConnectionService connectionService,
- final IndexSchemaCacheFactory indexSchemaCacheFactory,
- final UUID applicationId ) {
+ final CounterUtils counterUtils,
+ final AsyncEventService indexService,
+ final ManagerCache managerCache,
+ final MetricsFactory metricsFactory,
+ final EntityManagerFig entityManagerFig,
+ final GraphManagerFactory graphManagerFactory,
+ final CollectionService collectionService,
+ final ConnectionService connectionService,
+ final IndexSchemaCacheFactory indexSchemaCacheFactory,
+ final UUID applicationId ) {
this.entityManagerFig = entityManagerFig;
[39/44] usergrid git commit: Only execute read repair delete batch if
there are no rows to delete.
Posted by sn...@apache.org.
Only execute read repair delete batch if there are no rows to delete.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/48780f07
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/48780f07
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/48780f07
Branch: refs/heads/usergrid-1268-akka-211
Commit: 48780f07d1d94bc7b8a6934dbd5da3446102e8fc
Parents: 7143cba
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Jun 6 10:44:11 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Jun 6 10:44:11 2016 -0700
----------------------------------------------------------------------
.../impl/EntityCollectionManagerImpl.java | 22 +++++++++++++-------
1 file changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/48780f07/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 70b06ba..fcf1c6b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -385,23 +385,29 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
response.addEntity( expectedUnique.getField(), entity );
}
- deleteBatch.execute();
+ if ( deleteBatch.getRowCount() > 0 ) {
- // optionally sleep after read repair as some tasks immediately try to write after the delete
- if ( serializationFig.getReadRepairDelay() > 0 ){
+ deleteBatch.execute();
- try {
+ // optionally sleep after read repair as some tasks immediately try to write after the delete
+ if ( serializationFig.getReadRepairDelay() > 0 ){
- Thread.sleep(Math.min(serializationFig.getReadRepairDelay(), 200L));
+ try {
- } catch (InterruptedException e) {
+ Thread.sleep(Math.min(serializationFig.getReadRepairDelay(), 200L));
+
+ } catch (InterruptedException e) {
+
+ // do nothing if sleep fails; log and continue on
+ logger.warn("Sleep during unique value read repair failed.");
+ }
- // do nothing if sleep fails; log and continue on
- logger.warn("Sleep during unique value read repair failed.");
}
}
+
+
return response;
}
catch ( ConnectionException e ) {
[31/44] usergrid git commit: Merge commit 'refs/pull/532/head' of
github.com:apache/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge commit 'refs/pull/532/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a1cb1f5f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a1cb1f5f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a1cb1f5f
Branch: refs/heads/usergrid-1268-akka-211
Commit: a1cb1f5f45352a3dc7e7c0df91bb037e027d0ddd
Parents: 8976b41 ed4e67c
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jun 3 07:42:43 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jun 3 07:42:43 2016 -0700
----------------------------------------------------------------------
.../usergrid/security/shiro/ShiroCache.java | 80 ++++++++++++--------
1 file changed, 49 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
[29/44] usergrid git commit: Additional test.
Posted by sn...@apache.org.
Additional test.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/317714bc
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/317714bc
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/317714bc
Branch: refs/heads/usergrid-1268-akka-211
Commit: 317714bc4e5be27a220420c4df4ab281309ba086
Parents: e9228e1
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 3 08:30:42 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 3 08:30:42 2016 -0400
----------------------------------------------------------------------
.../queries/SelectMappingsQueryTest.java | 38 ++++++++++++++++++--
1 file changed, 36 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/317714bc/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
index 4a291b6..fd33c15 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -103,7 +103,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
* Field named testProp can be over-written by field named TESTPROP.
*/
@Test
- public void testFieldOverride() throws Exception {
+ public void testFieldOverride1() throws Exception {
String collectionName = "things";
@@ -119,7 +119,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
app().collection( collectionName ).post( entity );
refreshIndex();
- // testProp and TESTPROP should now have newValue
+ // testProp and TESTPROP should new be queryable by new value
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + newValue + "'" );
@@ -131,4 +131,38 @@ public class SelectMappingsQueryTest extends QueryTestBase {
things = app().collection( "things" ).get( params );
assertEquals( 1, things.getNumOfEntities() );
}
+
+ /**
+ * Field named testProp can be over-written by field named TESTPROP.
+ */
+ @Test
+ public void testFieldOverride2() throws Exception {
+
+ String collectionName = "things";
+
+ // create entity with TESTPROP=value
+ String value = RandomStringUtils.randomAlphabetic( 20 );
+ Entity entity = new Entity().withProp( "TESTPROP", value );
+ app().collection( collectionName ).post( entity );
+ refreshIndex();
+
+ // override with testProp=newValue
+ String newValue = RandomStringUtils.randomAlphabetic( 20 );
+ entity = new Entity().withProp( "testProp", newValue );
+ app().collection( collectionName ).post( entity );
+ refreshIndex();
+
+ // testProp and TESTPROP should new be queryable by new value
+
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testProp='" + newValue + "'" );
+ Collection things = this.app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+
+ params = new QueryParameters()
+ .setQuery( "select * where TESTPROP='" + newValue + "'" );
+ things = app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+ }
+
}
[12/44] usergrid git commit: Revert changes to initMgmtApp() and add
null-pointer check.
Posted by sn...@apache.org.
Revert changes to initMgmtApp() and add null-pointer check.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/12743f3e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/12743f3e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/12743f3e
Branch: refs/heads/usergrid-1268-akka-211
Commit: 12743f3e221f80df7eab89fda08325e9775f7ae9
Parents: de6de66
Author: Dave Johnson <sn...@apache.org>
Authored: Fri May 20 14:03:53 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri May 20 14:03:53 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 49 +++++---------------
1 file changed, 12 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/12743f3e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e057210..924dd10 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -175,12 +175,12 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
- if ( app != null ) {
+ if ( app != null && entityManager != null ) {
// we successfully fetched up the management app, cache it for a rainy day
managementAppEntityManager = entityManager;
- } else if ( managementAppEntityManager != null ) {
+ } else if ( entityManager == null && managementAppEntityManager != null ) {
// failed to fetch management app, use cached one
entityManager = managementAppEntityManager;
@@ -214,47 +214,22 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
-
private void initMgmtAppInternal() {
- Properties properties = cassandraService.getProperties();
-
- Integer maxRetries;
- try {
- Object maxRetriesObject = properties.get( MANAGEMENT_APP_MAX_RETRIES ).toString();
- maxRetries = Integer.parseInt( maxRetriesObject.toString() );
- } catch ( NumberFormatException nfe ) {
- maxRetries = 20;
- }
-
EntityManager em = getEntityManager(getManagementAppId());
+ indexService.queueInitializeApplicationIndex(CpNamingUtils.getApplicationScope(getManagementAppId()));
- int retryCount = 0;
-
- while ( managementApp != null && retryCount++ <= maxRetries ) {
-
- try {
- managementApp = em.getApplication();
-
- if ( managementApp == null ) {
-
- logger.warn( "Management application not found, attempting creation" );
-
- Map mgmtAppProps = new HashMap<String, Object>();
- mgmtAppProps.put( PROPERTY_NAME, CassandraService.MANAGEMENT_APPLICATION );
- em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps );
- managementApp = em.getApplication();
- }
-
- } catch ( Throwable t ) {
- logger.warn("Error getting or creating management application after " + retryCount + " retries", t);
+ try {
+ if ( em.getApplication() == null ) {
+ logger.info("Creating management application");
+ Map mgmtAppProps = new HashMap<String, Object>();
+ mgmtAppProps.put(PROPERTY_NAME, CassandraService.MANAGEMENT_APPLICATION);
+ em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps);
+ em.getApplication();
}
- }
-
- indexService.queueInitializeApplicationIndex(CpNamingUtils.getApplicationScope(getManagementAppId()));
- if ( managementApp == null ) {
- throw new RuntimeException("FATAL ERROR: Failed to get or create management app");
+ } catch (Exception ex) {
+ throw new RuntimeException("Fatal error creating management application", ex);
}
}
[33/44] usergrid git commit: Quick fix for Core tests: ensure
lockManager gets initialized.
Posted by sn...@apache.org.
Quick fix for Core tests: ensure lockManager gets initialized.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/17809acd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/17809acd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/17809acd
Branch: refs/heads/usergrid-1268-akka-211
Commit: 17809acdac0c66725dbcd7ad79b0240e4a7290b0
Parents: e6cf6ed
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 3 11:45:13 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 3 11:45:13 2016 -0400
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/CoreSchemaManager.java | 8 +++++++-
stack/core/src/test/resources/usergrid-test-context.xml | 1 +
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/17809acd/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
index c6c2d26..28f2c9f 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.persistence;
+import org.apache.usergrid.locking.LockManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,11 +38,13 @@ public class CoreSchemaManager implements SchemaManager {
private final Setup setup;
private final Cluster cluster;
+ private final LockManager lockManager;
- public CoreSchemaManager( final Setup setup, final Cluster cluster ) {
+ public CoreSchemaManager( final Setup setup, final Cluster cluster, Injector injector ) {
this.setup = setup;
this.cluster = cluster;
+ this.lockManager = injector.getInstance( LockManager.class );
}
@@ -49,6 +52,7 @@ public class CoreSchemaManager implements SchemaManager {
public void create() {
try {
setup.initSchema();
+ lockManager.setup();
}
catch ( Exception ex ) {
logger.error( "Could not setup usergrid core schema", ex );
@@ -62,6 +66,8 @@ public class CoreSchemaManager implements SchemaManager {
try {
setup.initSchema();
+ lockManager.setup();
+
setup.runDataMigration();
setup.initMgmtApp();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/17809acd/stack/core/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-test-context.xml b/stack/core/src/test/resources/usergrid-test-context.xml
index 6b10a7c..5572619 100644
--- a/stack/core/src/test/resources/usergrid-test-context.xml
+++ b/stack/core/src/test/resources/usergrid-test-context.xml
@@ -55,5 +55,6 @@
<bean id="coreManager" class="org.apache.usergrid.persistence.CoreSchemaManager">
<constructor-arg ref="setup"/>
<constructor-arg ref="cassandraCluster"/>
+ <constructor-arg ref="injector"/>
</bean>
</beans>
[18/44] usergrid git commit: Revert all changes to CpEntityManager
Posted by sn...@apache.org.
Revert all changes to CpEntityManager
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e299024f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e299024f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e299024f
Branch: refs/heads/usergrid-1268-akka-211
Commit: e299024ff75caaff3111c9e762671fb91166d50c
Parents: 7d2de99
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:26:27 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:26:27 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 27 +++++++++++++++-----
1 file changed, 20 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e299024f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index acdfa0f..13b5d1f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -36,7 +36,6 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.stream.Collectors;
-import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +48,24 @@ import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.AggregateCounter;
+import org.apache.usergrid.persistence.AggregateCounterSet;
+import org.apache.usergrid.persistence.CollectionRef;
+import org.apache.usergrid.persistence.ConnectedEntityRef;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Query.Level;
+import org.apache.usergrid.persistence.RelationManager;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.SimpleRoleRef;
+import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@ -214,9 +230,6 @@ public class CpEntityManager implements EntityManager {
private EntityCollectionManager ecm;
- private CpEntityManagerFactory emf;
-
-
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -229,8 +242,7 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CpEntityManagerFactory emf,
- final CassandraService cass,
+ public CpEntityManager( final CassandraService cass,
final CounterUtils counterUtils,
final AsyncEventService indexService,
final ManagerCache managerCache,
@@ -242,7 +254,6 @@ public class CpEntityManager implements EntityManager {
final IndexSchemaCacheFactory indexSchemaCacheFactory,
final UUID applicationId ) {
- this.emf = emf;
this.entityManagerFig = entityManagerFig;
Preconditions.checkNotNull( cass, "cass must not be null" );
@@ -259,6 +270,8 @@ public class CpEntityManager implements EntityManager {
this.connectionService = connectionService;
this.collectionService = collectionService;
+
+
this.managerCache = managerCache;
this.applicationId = applicationId;
this.indexService = indexService;
[08/44] usergrid git commit: Merge branch
'usergrid-1283-service-mgr-init' into usegrid-1283-mgmt-app-init
Posted by sn...@apache.org.
Merge branch 'usergrid-1283-service-mgr-init' into usegrid-1283-mgmt-app-init
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/de6de662
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/de6de662
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/de6de662
Branch: refs/heads/usergrid-1268-akka-211
Commit: de6de6629903f57df4faf5c3f20f5cea63f2bcec
Parents: e0c0c87 5f46341
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 18 09:23:54 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 18 09:23:54 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 66 +++++++++++++++++---
1 file changed, 59 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6de662/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b61be01,4028875..e057210
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -98,14 -102,15 +98,12 @@@ public class CpEntityManagerFactory imp
private Setup setup = null;
+ EntityManager managementAppEntityManager = null;
+
// cache of already instantiated entity managers
- private LoadingCache<UUID, EntityManager> entityManagers
- = CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() {
- public EntityManager load(UUID appId) { // no checked exception
- return _getEntityManager(appId);
- }
- });
+ private final String ENTITY_MANAGER_CACHE_SIZE = "entity.manager.cache.size";
+ private final LoadingCache<UUID, EntityManager> entityManagers;
-
-
-
private final ApplicationIdCache applicationIdCache;
//private final IndexSchemaCache indexSchemaCache;
@@@ -148,7 -148,62 +146,61 @@@
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
- initMgmtAppInternal();
-
+ int entityManagerCacheSize = 100;
+ try {
+ entityManagerCacheSize = Integer.parseInt(
+ cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e );
+ }
+
+ entityManagers = CacheBuilder.newBuilder()
+ .maximumSize(entityManagerCacheSize)
+ .build(new CacheLoader<UUID, EntityManager>() {
+
+ public EntityManager load( UUID appId ) { // no checked exception
+
+ // get entity manager and ensure it can get its own application
+
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Exception exception = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Exception e) {
+ exception = e;
+ }
+
+ // the management app is a special case
+
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+
+ if ( app != null ) {
+
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
+
+ } else if ( managementAppEntityManager != null ) {
+
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
+
+ } else {
+
+ // fetch failed and we have nothing cached, we must be bootstrapping
+ logger.info("managementAppEntityManager is null, bootstrapping in progress");
+ }
+
+ } else { // not the management app, so blow up if app cannot be fetched
+
+ if (app == null) {
+ throw new RuntimeException( "Error getting application " + appId, exception );
+ }
+ }
+
+ return entityManager;
+ }
+ });
}
[40/44] usergrid git commit: Merge commit 'refs/pull/534/head' of
github.com:apache/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge commit 'refs/pull/534/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/38909ada
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/38909ada
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/38909ada
Branch: refs/heads/usergrid-1268-akka-211
Commit: 38909adaa0e6332f3fdb198a5bd394009d7213d4
Parents: cc7a867 48780f0
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Jun 6 10:51:59 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Jun 6 10:51:59 2016 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../corepersistence/EntityManagerFig.java | 2 +-
.../asyncevents/AsyncEventService.java | 6 +-
.../asyncevents/AsyncEventServiceImpl.java | 11 +-
.../asyncevents/EventBuilder.java | 6 +-
.../asyncevents/EventBuilderImpl.java | 46 ++++---
.../model/DeIndexOldVersionsEvent.java | 12 +-
.../corepersistence/index/IndexService.java | 23 +++-
.../corepersistence/index/IndexServiceImpl.java | 86 +++++++------
.../collection/EntityCollectionManager.java | 10 +-
.../impl/EntityCollectionManagerImpl.java | 40 +++++-
.../serialization/SerializationFig.java | 5 +
.../serialization/impl/LogEntryIterator.java | 128 +++++++++++++++++++
.../usergrid/persistence/index/EntityIndex.java | 5 +-
.../usergrid/persistence/index/IndexFig.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 64 +++-------
.../persistence/index/impl/EntityIndexTest.java | 41 ------
17 files changed, 320 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
[41/44] usergrid git commit: bug fix - When cursor queryparam is
passed in with an empty value then throw 400.
Posted by sn...@apache.org.
bug fix - When cursor queryparam is passed in with an empty value then throw 400.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0eaea989
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0eaea989
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0eaea989
Branch: refs/heads/usergrid-1268-akka-211
Commit: 0eaea9890b5149f2e7203a90c1dd37a886bbd12c
Parents: 38909ad
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Mon Jun 6 13:55:41 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Mon Jun 6 13:55:41 2016 -0700
----------------------------------------------------------------------
.../pipeline/cursor/RequestCursor.java | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0eaea989/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
index dc6ae71..acd6e25 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
@@ -20,17 +20,15 @@
package org.apache.usergrid.corepersistence.pipeline.cursor;
-import java.util.Base64;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.fasterxml.jackson.core.Base64Variant;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* A cursor that has been passed in with our request. Adds utils for parsing values
@@ -76,13 +74,13 @@ public class RequestCursor {
* Deserialize from the cursor as json nodes
*/
private Map<Integer, JsonNode> fromCursor( final String cursor ) throws CursorParseException {
- try {
-
-
+ if(cursor.isEmpty()){
+ throw new IllegalArgumentException("cursor cannot be empty");
+ }
+ try {
JsonNode jsonNode = CursorSerializerUtil.fromString( cursor );
-
Preconditions
.checkArgument( jsonNode.size() <= MAX_CURSOR_COUNT, " You cannot have more than " + MAX_CURSOR_COUNT + " cursors" );
[14/44] usergrid git commit: Improvements to error handling in cache
key generation
Posted by sn...@apache.org.
Improvements to error handling in cache key generation
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ed4e67c8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ed4e67c8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ed4e67c8
Branch: refs/heads/usergrid-1268-akka-211
Commit: ed4e67c855a1d1062d0cfa5976d05db6b044f11a
Parents: aae8fdf
Author: Dave Johnson <sn...@apache.org>
Authored: Mon May 23 11:41:44 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon May 23 11:41:44 2016 -0400
----------------------------------------------------------------------
.../usergrid/security/shiro/ShiroCache.java | 80 ++++++++++++--------
1 file changed, 49 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ed4e67c8/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
index b4803b1..6c5ac6c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
@@ -184,51 +184,69 @@ public class ShiroCache<K, V> implements Cache<K,V> {
String ret = null;
- final String typeName = typeRef.getType().getTypeName();
+ Throwable throwable = null;
- if ( key instanceof SimplePrincipalCollection) {
+ String errorMessage = null;
- SimplePrincipalCollection spc = (SimplePrincipalCollection)key;
+ try {
- if ( spc.getPrimaryPrincipal() instanceof UserPrincipal) {
+ final String typeName = typeRef.getType().getTypeName();
- // principal is a user, use UUID as cache key
- UserPrincipal p = (UserPrincipal) spc.getPrimaryPrincipal();
- ret = p.getUser().getUuid().toString() + "_" + typeName;
- }
+ if (key instanceof SimplePrincipalCollection) {
+
+ SimplePrincipalCollection spc = (SimplePrincipalCollection) key;
+
+ if (spc.getPrimaryPrincipal() instanceof UserPrincipal) {
+
+ // principal is a user, use UUID as cache key
+ UserPrincipal p = (UserPrincipal) spc.getPrimaryPrincipal();
+ ret = p.getUser().getUuid().toString() + "_" + typeName;
+
+ } else if (spc.getPrimaryPrincipal() instanceof PrincipalIdentifier) {
- else if ( spc.getPrimaryPrincipal() instanceof PrincipalIdentifier ) {
+ // principal is not user, try to get something unique as cache key
+ PrincipalIdentifier p = (PrincipalIdentifier) spc.getPrimaryPrincipal();
+ if (p.getAccessTokenCredentials() != null) {
+ ret = p.getAccessTokenCredentials().getToken() + "_" + typeName;
+ } else {
+ ret = p.getApplicationId() + "_" + typeName;
+ }
- // principal is not user, try to get something unique as cache key
- PrincipalIdentifier p = (PrincipalIdentifier) spc.getPrimaryPrincipal();
- if (p.getAccessTokenCredentials() != null) {
- ret = p.getAccessTokenCredentials().getToken() + "_" + typeName;
} else {
- ret = p.getApplicationId() + "_" + typeName;
+ errorMessage = "Unknown principal type: " + key.getClass().getSimpleName();
}
- }
- } else if ( key instanceof ApplicationGuestPrincipal ) {
- ApplicationGuestPrincipal agp = (ApplicationGuestPrincipal) key;
- ret = agp.getApplicationId() + "_" + typeName;
+ } else if (key instanceof ApplicationGuestPrincipal) {
+ ApplicationGuestPrincipal agp = (ApplicationGuestPrincipal) key;
+ ret = agp.getApplicationId() + "_" + typeName;
+
+ } else if (key instanceof ApplicationPrincipal) {
+ ApplicationPrincipal ap = (ApplicationPrincipal) key;
+ ret = ap.getApplicationId() + "_" + typeName;
+
+ } else if (key instanceof OrganizationPrincipal) {
+ OrganizationPrincipal op = (OrganizationPrincipal) key;
+ ret = op.getOrganizationId() + "_" + typeName;
- } else if ( key instanceof ApplicationPrincipal ) {
- ApplicationPrincipal ap = (ApplicationPrincipal) key;
- ret = ap.getApplicationId() + "_" + typeName;
+ } else if (key instanceof UserPrincipal) {
+ UserPrincipal up = (UserPrincipal) key;
+ ret = up.getUser().getUuid() + "_" + typeName;
- } else if ( key instanceof OrganizationPrincipal ) {
- OrganizationPrincipal op = (OrganizationPrincipal) key;
- ret = op.getOrganizationId() + "_" + typeName;
+ } else {
+ errorMessage = "Unknown key type: " + key.getClass().getSimpleName();
+ }
+
+ } catch ( Throwable t ) {
+ throwable = t;
+ }
- } else if ( key instanceof UserPrincipal ) {
- UserPrincipal up = (UserPrincipal)key;
- ret = up.getUser().getUuid() + "_" + typeName;
+ if ( throwable != null ) {
+ errorMessage = "Error generating cache key for key type " + key.getClass().getSimpleName();
+ throw new CacheException( errorMessage, throwable );
}
- if ( ret == null) {
- String msg = "Unknown key type: " + key.getClass().getSimpleName();
- logger.error(msg);
- throw new RuntimeException(msg);
+ if ( ret == null ) {
+ throw new CacheException( errorMessage );
}
return ret;
[28/44] usergrid git commit: Remove unneeded properties,
log when mgmt app lookup fails, minor refactoring
Posted by sn...@apache.org.
Remove unneeded properties, log when mgmt app lookup fails, minor refactoring
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b8e5460
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b8e5460
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b8e5460
Branch: refs/heads/usergrid-1268-akka-211
Commit: 4b8e5460cb93fb65ee54282e93887668bf7d6f8a
Parents: 59c538a
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 3 07:31:46 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 3 07:31:46 2016 -0400
----------------------------------------------------------------------
.../src/main/resources/usergrid-default.properties | 7 -------
.../corepersistence/CpEntityManagerFactory.java | 14 +++++++-------
.../exception/CollectionRuntimeException.java | 2 +-
3 files changed, 8 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b8e5460/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 1e22d6a..0fc31ef 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -195,13 +195,6 @@ cassandra.lock.writecl=LOCAL_QUORUM
#
#cassandra.useSocketKeepalive=false
-# Number of times to retry creation of lock keyspace and column family
-cassandra.lock.init.retries = 100;
-
-# Interval between lock keyspace creation attempts (in milliseconds)
-cassandra.lock.init.interval = 1000;
-
-
##################### Async Threadpool Settings #####################
# Set the number of threads available in the Rx Async Thread Pool
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b8e5460/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 622944b..18df205 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -69,7 +69,6 @@ import java.util.*;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
-import static org.apache.usergrid.persistence.Schema.initLock;
/**
@@ -178,26 +177,27 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
} else if ( managementAppEntityManager != null ) {
// failed to fetch management app, use cached one
entityManager = managementAppEntityManager;
+ logger.error("Failed to fetch management app");
}
}
// missing keyspace means we have not done bootstrap yet
- final boolean missingKeyspace;
+ final boolean isBootstrapping;
if ( throwable instanceof CollectionRuntimeException ) {
CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
- missingKeyspace = cre.isMissingKeyspace();
+ isBootstrapping = cre.isBootstrapping();
} else {
- missingKeyspace = false;
+ isBootstrapping = false;
}
// work around for https://issues.apache.org/jira/browse/USERGRID-1291
- // throw exception so that we do not cache
+ // throw exception so that we do not cache
// TODO: determine how application name can intermittently be null
if ( app != null && app.getName() == null ) {
throw new RuntimeException( "Name is null for application " + appId, throwable );
}
- if ( app == null && !missingKeyspace ) {
+ if ( app == null && !isBootstrapping ) {
throw new RuntimeException( "Error getting application " + appId, throwable );
} // else keyspace is missing because setup/bootstrap not done yet
@@ -241,7 +241,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
if ( t instanceof CollectionRuntimeException ) {
CollectionRuntimeException cre = (CollectionRuntimeException)t;
- if ( cre.isMissingKeyspace() ) {
+ if ( cre.isBootstrapping() ) {
// we're bootstrapping, ignore this and continue
bootstrapping = true;
break;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b8e5460/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
index f27e2d7..8aa2a7a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
@@ -57,7 +57,7 @@ public class CollectionRuntimeException extends RuntimeException {
this.applicationScope = scope;
}
- public boolean isMissingKeyspace() {
+ public boolean isBootstrapping() {
if ( getCause() instanceof BadRequestException ) {
BadRequestException bre = (BadRequestException)getCause();
String msg = bre.getMessage();
[02/44] usergrid git commit: Added test and cleaned up imports that
proves that the class cast issue will no longer occur.
Posted by sn...@apache.org.
Added test and cleaned up imports that proves that the class cast issue will no longer occur.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0f13650f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0f13650f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0f13650f
Branch: refs/heads/usergrid-1268-akka-211
Commit: 0f13650f500809b2d044568570dbf11f4a57f70c
Parents: c46e1b6
Author: George Reyes <gr...@apache.org>
Authored: Thu May 5 09:29:03 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Fri May 6 15:38:34 2016 -0700
----------------------------------------------------------------------
.../collection/CollectionsResourceIT.java | 82 +++++++++++++++-----
1 file changed, 64 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0f13650f/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index db07c3f..7afc87a 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -17,12 +17,28 @@
package org.apache.usergrid.rest.applications.collection;
-import com.fasterxml.jackson.databind.JsonNode;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.ClientErrorException;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.commons.lang.RandomStringUtils;
+
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.entities.Application;
-
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.test.resource.AbstractRestIT;
import org.apache.usergrid.rest.test.resource.model.ApiResponse;
import org.apache.usergrid.rest.test.resource.model.Collection;
@@ -30,24 +46,16 @@ import org.apache.usergrid.rest.test.resource.model.Credentials;
import org.apache.usergrid.rest.test.resource.model.Entity;
import org.apache.usergrid.rest.test.resource.model.QueryParameters;
import org.apache.usergrid.rest.test.resource.model.Token;
-import org.apache.usergrid.services.ServiceParameter;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.BadRequestException;
-import javax.ws.rs.ClientErrorException;
-import javax.ws.rs.client.Invocation;
-import javax.ws.rs.core.MediaType;
-
-import java.io.IOException;
-import java.util.*;
-import org.apache.commons.lang.NullArgumentException;
+import com.fasterxml.jackson.databind.JsonNode;
-import static org.junit.Assert.*;
+import static junit.framework.TestCase.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
@@ -1096,4 +1104,42 @@ public class CollectionsResourceIT extends AbstractRestIT {
assertEquals( 0, connectionsByQuery.getNumOfEntities() );
}
+ @Test
+ public void testBeingAbleToRetreiveMigratedValues() throws Exception {
+
+
+ Entity notifier = new Entity().chainPut("name", "mynotifier").chainPut("provider", "noop");
+
+ ApiResponse notifierNode = this.pathResource(getOrgAppPath("notifier")).post(ApiResponse.class,notifier);
+
+ UUID uuid = UUIDUtils.newTimeUUID();
+ // create user
+
+ Map payloads = new HashMap<>( );
+ payloads.put( "mynotifier","hello world" );
+
+ Map statistics = new HashMap<>( );
+ statistics.put( "sent",1 );
+ statistics.put( "errors",0 );
+
+ Entity payload = new Entity();
+ payload.put("debug", false);
+ payload.put( "expectedCount",0 );
+ payload.put( "finished",1438279671229L);
+ payload.put( "payloads",payloads);
+ payload.put( "priority","normal");
+ payload.put( "state","FINISHED");
+ payload.put( "statistics",statistics);
+
+
+
+
+ ApiResponse user = this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).put(null,payload );
+ this.refreshIndex();
+
+ Collection user2 = this.app().collection("notifications").get();
+
+ assertEquals(1,user2.getNumOfEntities());
+
+ }
}
[09/44] usergrid git commit: Initial support for select mappings with
entity object fields (i.e. nested fields)
Posted by sn...@apache.org.
Initial support for select mappings with entity object fields (i.e. nested fields)
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/596e0fc5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/596e0fc5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/596e0fc5
Branch: refs/heads/usergrid-1268-akka-211
Commit: 596e0fc5cef2eb1ae4aba40343bf27e73aee86d6
Parents: 7fdca3d
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 18 17:24:25 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 18 17:24:25 2016 -0400
----------------------------------------------------------------------
.../read/search/CandidateEntityFilter.java | 59 ++++++++++++++-
.../apache/usergrid/persistence/IndexIT.java | 80 ++++++++++++++++++++
2 files changed, 135 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/596e0fc5/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index d47e96c..261259b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -27,7 +27,9 @@ import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.apache.usergrid.persistence.model.field.DistanceField;
import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.EntityObjectField;
import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -123,12 +125,26 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
if (mappings.size() > 0) {
Map<String,Field> fieldMap = new HashMap<String, Field>(mappings.size());
rx.Observable.from(mappings)
- .filter(mapping -> entity.getFieldMap().containsKey(mapping.getSourceFieldName()))
+
+ .filter(mapping -> {
+ if ( entity.getFieldMap().containsKey(mapping.getSourceFieldName())) {
+ return true;
+ }
+ String[] parts = mapping.getSourceFieldName().split("\\.");
+ return nestedFieldCheck( parts, entity.getFieldMap() );
+ })
+
.doOnNext(mapping -> {
Field field = entity.getField(mapping.getSourceFieldName());
- field.setName(mapping.getTargetFieldName());
- fieldMap.put(mapping.getTargetFieldName(),field);
- }).toBlocking().last();
+ if ( field != null ) {
+ field.setName( mapping.getTargetFieldName() );
+ fieldMap.put( mapping.getTargetFieldName(), field );
+ } else {
+ String[] parts = mapping.getSourceFieldName().split("\\.");
+ nestedFieldSet( fieldMap, parts, entity.getFieldMap() );
+ }
+ }).toBlocking().lastOrDefault(null);
+
entity.setFieldMap(fieldMap);
}
return entityFilterResult;
@@ -144,6 +160,41 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
}
+ private void nestedFieldSet( Map<String, Field> result, String[] parts, Map<String, Field> fieldMap) {
+ if ( parts.length > 0 ) {
+ if ( fieldMap.containsKey( parts[0] )) {
+ Field field = fieldMap.get( parts[0] );
+ if ( field instanceof EntityObjectField ) {
+ EntityObjectField eof = (EntityObjectField)field;
+ if ( result.get( parts[0] ) == null ) {
+ result.put( parts[0], new EntityObjectField( parts[0], new EntityObject() ) );
+ }
+ nestedFieldSet(
+ ((EntityObjectField)result.get( parts[0] )).getValue().getFieldMap(),
+ Arrays.copyOfRange(parts, 1, parts.length),
+ eof.getValue().getFieldMap());
+ } else {
+ result.put( parts[0], field );
+ }
+ }
+ }
+ }
+
+
+ private boolean nestedFieldCheck( String[] parts, Map<String, Field> fieldMap) {
+ if ( parts.length > 0 ) {
+ if ( fieldMap.containsKey( parts[0] )) {
+ Field field = fieldMap.get( parts[0] );
+ if ( field instanceof EntityObjectField ) {
+ EntityObjectField eof = (EntityObjectField)field;
+ return nestedFieldCheck( Arrays.copyOfRange(parts, 1, parts.length), eof.getValue().getFieldMap());
+ } else {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/596e0fc5/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index f4aa204..7e38f17 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.persistence;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
@@ -461,4 +462,83 @@ public class IndexIT extends AbstractCoreIT {
}
+
+ @Test
+ public void testSelectMappings() throws Exception {
+
+ UUID applicationId = app.getId();
+
+ EntityManager em = setup.getEmf().getEntityManager(applicationId);
+
+ Map<String, Object> entity1 = new HashMap<String, Object>() {{
+ put("name","name_1");
+ put("status", "pickled");
+ put("data", new HashMap() {{
+ put("xfactor", 5.1);
+ put("rando", "bar");
+ }});
+ }};
+ em.create("names", entity1);
+
+ Map<String, Object> entity2 = new HashMap<String, Object>() {{
+ put("name","name_2");
+ put("status", "pickled");
+ put("data", new HashMap() {{
+ put("xfactor", 5.1);
+ put("rando", "bar");
+ }});
+ }};
+ em.create("names", entity2);
+
+ app.refreshIndex();
+
+ {
+ Query query = Query.fromQL("select status where status = 'pickled'");
+ Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+ assertTrue(r.getEntities() != null && r.getEntities().size() == 2);
+ Entity first = r.getEntities().get(0);
+ assertTrue(first.getDynamicProperties().size() == 2);
+ }
+
+ {
+ Query query = Query.fromQL( "select status, data.rando where data.rando = 'bar'" );
+ Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+ assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+
+ Entity first = r.getEntities().get( 0 );
+
+ assertNotNull( first.getProperty("status") );
+ assertEquals( first.getProperty("status"), "pickled" );
+
+ assertNotNull( first.getProperty("data") );
+ assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
+
+ assertTrue( first.getDynamicProperties().size() == 3 );
+ }
+
+ {
+ // query for only one bogus field name should return empty entities
+ Query query = Query.fromQL( "select data.rando where status = 'pickled'" );
+ Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+ assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+
+ Entity first = r.getEntities().get( 0 );
+
+ assertNotNull( first.getProperty("data") );
+ assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
+
+ assertTrue( first.getDynamicProperties().size() == 2 );
+ }
+
+ {
+ // query for only one bogus field name should return empty entities
+ Query query = Query.fromQL( "select data.bogusfieldname where status = 'pickled'" );
+ Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+ assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+ Entity first = r.getEntities().get( 0 );
+ assertTrue( first.getDynamicProperties().size() == 1 );
+ }
+
+ }
+
}
[36/44] usergrid git commit: better logging
Posted by sn...@apache.org.
better logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b3e60c93
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b3e60c93
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b3e60c93
Branch: refs/heads/usergrid-1268-akka-211
Commit: b3e60c93bca7b463121991dca2d48e553c337aba
Parents: 66bb5cd
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Jun 6 12:20:15 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Jun 6 12:20:15 2016 -0400
----------------------------------------------------------------------
.../org/apache/usergrid/locking/cassandra/AstyanaxLockImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b3e60c93/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockImpl.java
index 1399d16..d06d579 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockImpl.java
@@ -67,7 +67,7 @@ public class AstyanaxLockImpl implements Lock {
count.incrementAndGet();
} catch (Exception e) {
- throw new UGLockException("Unable to acquire lock with id: " + lock.getLockId());
+ throw new UGLockException("Unable to acquire lock with id: " + lock.getLockId(), e);
}
}
[37/44] usergrid git commit: core schema manager needs injector now
Posted by sn...@apache.org.
core schema manager needs injector now
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fcb347c7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fcb347c7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fcb347c7
Branch: refs/heads/usergrid-1268-akka-211
Commit: fcb347c763b1a9027e8261cb54671f6d92f61da3
Parents: b3e60c9
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Jun 6 12:20:36 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Jun 6 12:20:36 2016 -0400
----------------------------------------------------------------------
stack/rest/src/test/resources/usergrid-test-context.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcb347c7/stack/rest/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-test-context.xml b/stack/rest/src/test/resources/usergrid-test-context.xml
index a9fc5a5..76b9775 100644
--- a/stack/rest/src/test/resources/usergrid-test-context.xml
+++ b/stack/rest/src/test/resources/usergrid-test-context.xml
@@ -45,7 +45,6 @@
</bean>
<bean id="setup" class="org.apache.usergrid.corepersistence.CpSetup">
-
<constructor-arg ref="entityManagerFactory"/>
<constructor-arg ref="cassandraService"/>
<constructor-arg ref="injector"/>
@@ -55,6 +54,7 @@
<bean id="coreManager" class="org.apache.usergrid.persistence.CoreSchemaManager">
<constructor-arg ref="setup"/>
<constructor-arg ref="cassandraCluster"/>
+ <constructor-arg ref="injector"/>
</bean>
</beans>
[24/44] usergrid git commit: Move LockManager setup into normal setup
regime, don't cache EntityManagers with null application names,
and add logging to help debug issue USERGRID-1283.
Posted by sn...@apache.org.
Move LockManager setup into normal setup regime, don't cache EntityManagers with null application names, and add logging to help debug issue USERGRID-1283.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/59c538aa
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/59c538aa
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/59c538aa
Branch: refs/heads/usergrid-1268-akka-211
Commit: 59c538aa8a6becf90573c815e68347036f1aca44
Parents: 61a35a0
Author: Dave Johnson <sn...@apache.org>
Authored: Tue May 31 09:34:48 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue May 31 09:34:48 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 13 +++++-
.../apache/usergrid/locking/LockManager.java | 5 +++
.../cassandra/AstyanaxLockManagerImpl.java | 42 ++++++--------------
.../locking/noop/NoOpLockManagerImpl.java | 5 +++
.../usergrid/services/ServiceManager.java | 8 ++++
.../services/ServiceManagerFactory.java | 12 ++++++
6 files changed, 54 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index ee28765..622944b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -33,6 +33,7 @@ import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
+import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
@@ -107,6 +108,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final ConnectionService connectionService;
private final GraphManagerFactory graphManagerFactory;
private final IndexSchemaCacheFactory indexSchemaCacheFactory;
+ private final LockManager lockManager;
public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries";
public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval";
@@ -127,6 +129,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.collectionService = injector.getInstance( CollectionService.class );
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
+ this.lockManager = injector.getInstance( LockManager.class );
Properties properties = cassandraService.getProperties();
@@ -166,7 +169,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
// the management app is a special case
-
if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
if ( app != null ) {
@@ -179,6 +181,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
}
+ // missing keyspace means we have not done bootstrap yet
final boolean missingKeyspace;
if ( throwable instanceof CollectionRuntimeException ) {
CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
@@ -187,6 +190,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
missingKeyspace = false;
}
+ // work around for https://issues.apache.org/jira/browse/USERGRID-1291
+ // throw exception so that we do not cache
+ // TODO: determine how application name can intermittently be null
+ if ( app != null && app.getName() == null ) {
+ throw new RuntimeException( "Name is null for application " + appId, throwable );
+ }
+
if ( app == null && !missingKeyspace ) {
throw new RuntimeException( "Error getting application " + appId, throwable );
@@ -645,6 +655,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public void setup() throws Exception {
getSetup().initSchema();
+ lockManager.setup();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java b/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java
index a2d37ab..4ed41d8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java
@@ -38,4 +38,9 @@ public interface LockManager {
* @throws UGLockException if the lock cannot be acquired
*/
public Lock createLock( final UUID applicationId, final String... path );
+
+ /**
+ * Setup lock persistence mechanism.
+ */
+ public void setup();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 6acce47..90b9d57 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -50,6 +50,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
private final CassandraFig cassandraFig;
+ private final CassandraCluster cassandraCluster;
private Keyspace keyspace;
private ColumnFamily columnFamily;
private static final int MINIMUM_LOCK_EXPIRATION = 60000; // 1 minute
@@ -60,38 +61,19 @@ public class AstyanaxLockManagerImpl implements LockManager {
CassandraCluster cassandraCluster ) throws ConnectionException {
this.cassandraFig = cassandraFig;
+ this.cassandraCluster = cassandraCluster;
+ }
- // hold up construction until we can create the column family
- int maxRetries = cassandraFig.getLockManagerInitRetries();
- int retries = 0;
- boolean famReady = false;
- Set<Class> seenBefore = new HashSet<>(10);
- while ( !famReady && retries++ < maxRetries ) {
- try {
- keyspace = cassandraCluster.getLocksKeyspace();
- createLocksKeyspace();
- columnFamily = createLocksColumnFamily();
- famReady = true;
-
- } catch ( Throwable t ) {
- final String msg;
- if ( t instanceof PoolTimeoutException || t instanceof NoAvailableHostsException) {
- msg = retries + ": Cannot connect to Cassandra (" + t.getClass().getSimpleName() + ")";
- } else {
- msg = retries + ": Error (" + t.getClass().getSimpleName() + ") tries=" + retries;
- }
- if ( !seenBefore.contains( t.getClass() ) ) {
- logger.error( msg, t );
- } else {
- logger.error( msg );
- }
- seenBefore.add( t.getClass() );
- try {
- Thread.sleep( cassandraFig.getLockManagerInitInterval() );
- } catch (InterruptedException ignored) {}
- }
- }
+ @Override
+ public void setup() {
+ try {
+ keyspace = cassandraCluster.getLocksKeyspace();
+ createLocksKeyspace();
+ columnFamily = createLocksColumnFamily();
+ } catch (ConnectionException e) {
+ throw new RuntimeException( "Error setting up locks keyspace and column family", e );
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java
index 6174890..ff14031 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java
@@ -38,4 +38,9 @@ public class NoOpLockManagerImpl implements LockManager {
public Lock createLock( UUID applicationId, String... path ) {
return new NoOpLockImpl();
}
+
+ @Override
+ public void setup() {
+ // no op
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index a9892f5..04e00e0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@ -96,6 +96,14 @@ public class ServiceManager {
this.qm = qm;
this.properties = properties;
+ // additional logging to help debug https://issues.apache.org/jira/browse/USERGRID-1291
+ if ( em == null ) {
+ logger.error("EntityManager is null");
+ }
+ if ( qm == null ) {
+ logger.error("QueueManager is null");
+ }
+
if ( em != null ) {
try {
application = em.getApplication();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java
index 5274336..2425b95 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java
@@ -23,6 +23,8 @@ import java.util.UUID;
import com.google.inject.Injector;
import org.apache.usergrid.locking.Lock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
@@ -35,6 +37,7 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
public class ServiceManagerFactory implements ApplicationContextAware {
+ private static final Logger logger = LoggerFactory.getLogger( ServiceManagerFactory.class );
private ApplicationContext applicationContext;
@@ -59,6 +62,15 @@ public class ServiceManagerFactory implements ApplicationContextAware {
public ServiceManager getServiceManager( UUID applicationId ) {
+
+ // additional logging to help debug https://issues.apache.org/jira/browse/USERGRID-1291
+ if ( emf == null ) {
+ logger.error("EntityManagerFactory is null");
+ }
+ if ( qmf == null ) {
+ logger.error("QueueManagerFactory is null");
+ }
+
EntityManager em = null;
if ( emf != null ) {
em = emf.getEntityManager( applicationId );
[04/44] usergrid git commit: Removed the type checking for integers
since updateStatistics is only used once for guaranteed longs.
Posted by sn...@apache.org.
Removed the type checking for integers since updateStatistics is only used once for guaranteed longs.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9edfda62
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9edfda62
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9edfda62
Branch: refs/heads/usergrid-1268-akka-211
Commit: 9edfda620eefae7ce74a54ac1eff24b94d02094c
Parents: 987ffdc
Author: George Reyes <gr...@apache.org>
Authored: Mon May 9 12:19:28 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Mon May 9 12:19:28 2016 -0700
----------------------------------------------------------------------
.../usergrid/persistence/entities/Notification.java | 11 ++++++-----
.../applications/collection/CollectionsResourceIT.java | 2 +-
2 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9edfda62/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
index bb2e03f..c4867c3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
@@ -308,12 +308,13 @@ public class Notification extends TypedEntity {
this.statistics.put("sent", sent);
this.statistics.put("errors", errors);
} else {
- if(this.statistics.get( "sent" ) instanceof Integer){
- this.statistics.put( "sent", sent + (Integer) this.statistics.get( "sent" ) );
- this.statistics.put( "errors", errors + (Integer) this.statistics.get( "errors" ) );
- }
- else if (this.statistics.get( "sent" ) instanceof Long ) {
+ //Don't need to account for integers here because this is only called internally
+ //We won't ever need to call updateStatistics for a postedNotification as that only happens once
+ //after the notification is completed.
+ if (this.statistics.get( "sent" ) instanceof Long ) {
this.statistics.put( "sent", sent + (Long) this.statistics.get( "sent" ) );
+ }
+ if( this.statistics.get( "errors" ) instanceof Long){
this.statistics.put( "errors", errors + (Long) this.statistics.get( "errors" ) );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9edfda62/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index 0908f5c..690cec2 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -1118,7 +1118,7 @@ public class CollectionsResourceIT extends AbstractRestIT {
Map statistics = new HashMap<>( );
statistics.put( "sent",1 );
- statistics.put( "errors",0 );
+ statistics.put( "errors",2 );
Entity payload = new Entity();
payload.put("debug", false);
[15/44] usergrid git commit: Merge branch 'release-2.1.1' into
usegrid-1283-mgmt-app-init
Posted by sn...@apache.org.
Merge branch 'release-2.1.1' into usegrid-1283-mgmt-app-init
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8ebc28df
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8ebc28df
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8ebc28df
Branch: refs/heads/usergrid-1268-akka-211
Commit: 8ebc28dfd8e0d1abe4edec8a3941175238400ea4
Parents: 12743f3 aae8fdf
Author: Dave Johnson <sn...@apache.org>
Authored: Tue May 24 10:01:25 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue May 24 10:01:25 2016 -0400
----------------------------------------------------------------------
.../applications/ApplicationResourceIT.java | 5 +--
.../usergrid/security/shiro/ShiroCache.java | 44 +++++++++++++++++++-
2 files changed, 44 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[06/44] usergrid git commit: Move the initial get of management app
to the CpEntityManagerFactory with retries,
and caching for the management app itself.
Posted by sn...@apache.org.
Move the initial get of management app to the CpEntityManagerFactory with retries, and caching for the management app itself.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e0c0c875
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e0c0c875
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e0c0c875
Branch: refs/heads/usergrid-1268-akka-211
Commit: e0c0c875271cda47f9baf9072f029a92921fd1be
Parents: 7fdca3d
Author: Dave Johnson <sn...@apache.org>
Authored: Mon May 16 16:10:07 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon May 16 16:10:07 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 52 ++++-----
.../corepersistence/CpEntityManagerFactory.java | 112 ++++++++++++++-----
.../usergrid/services/ServiceManager.java | 59 ++--------
3 files changed, 113 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0c0c875/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 68f5d71..c8f3253 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -36,6 +36,7 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.stream.Collectors;
+import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,24 +49,7 @@ import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.AggregateCounter;
-import org.apache.usergrid.persistence.AggregateCounterSet;
-import org.apache.usergrid.persistence.CollectionRef;
-import org.apache.usergrid.persistence.ConnectedEntityRef;
-import org.apache.usergrid.persistence.ConnectionRef;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Query.Level;
-import org.apache.usergrid.persistence.RelationManager;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.SimpleRoleRef;
-import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@ -230,6 +214,9 @@ public class CpEntityManager implements EntityManager {
private EntityCollectionManager ecm;
+ private CpEntityManagerFactory emf;
+
+
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -242,18 +229,20 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager( final CassandraService cass,
- final CounterUtils counterUtils,
- final AsyncEventService indexService,
- final ManagerCache managerCache,
- final MetricsFactory metricsFactory,
- final EntityManagerFig entityManagerFig,
- final GraphManagerFactory graphManagerFactory,
- final CollectionService collectionService,
- final ConnectionService connectionService,
- final IndexSchemaCacheFactory indexSchemaCacheFactory,
- final UUID applicationId ) {
-
+ public CpEntityManager(final CpEntityManagerFactory emf,
+ final CassandraService cass,
+ final CounterUtils counterUtils,
+ final AsyncEventService indexService,
+ final ManagerCache managerCache,
+ final MetricsFactory metricsFactory,
+ final EntityManagerFig entityManagerFig,
+ final GraphManagerFactory graphManagerFactory,
+ final CollectionService collectionService,
+ final ConnectionService connectionService,
+ final IndexSchemaCacheFactory indexSchemaCacheFactory,
+ final UUID applicationId ) {
+
+ this.emf = emf;
this.entityManagerFig = entityManagerFig;
Preconditions.checkNotNull( cass, "cass must not be null" );
@@ -270,8 +259,6 @@ public class CpEntityManager implements EntityManager {
this.connectionService = connectionService;
this.collectionService = collectionService;
-
-
this.managerCache = managerCache;
this.applicationId = applicationId;
this.indexService = indexService;
@@ -755,6 +742,9 @@ public class CpEntityManager implements EntityManager {
@Override
public Application getApplication() throws Exception {
if ( application == null ) {
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( applicationId )) {
+ return emf.getManagementApplication();
+ }
application = get( applicationId, Application.class );
}
return application;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0c0c875/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 91a936d..b61be01 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -16,11 +16,7 @@
package org.apache.usergrid.corepersistence;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.UUID;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -113,6 +109,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final ApplicationIdCache applicationIdCache;
//private final IndexSchemaCache indexSchemaCache;
+ Application managementApp = null;
+
private ManagerCache managerCache;
private CassandraService cassandraService;
@@ -126,8 +124,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final GraphManagerFactory graphManagerFactory;
private final IndexSchemaCacheFactory indexSchemaCacheFactory;
- public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
- final Injector injector ) {
+ public static final String MANAGEMENT_APP_MAX_RETRIES= "management.app.max.retries";
+
+
+ public CpEntityManagerFactory(
+ final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector ) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
@@ -142,12 +143,12 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
- //this line always needs to be last due to the temporary cicular dependency until spring is removed
+ // this line always needs to be last due to the temporary circular dependency until spring is removed
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
-
+ initMgmtAppInternal();
}
@@ -164,21 +165,61 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private void initMgmtAppInternal() {
+ Properties properties = cassandraService.getProperties();
+
+ Integer maxRetries;
+ try {
+ Object maxRetriesObject = properties.get( MANAGEMENT_APP_MAX_RETRIES ).toString();
+ maxRetries = Integer.parseInt( maxRetriesObject.toString() );
+ } catch ( NumberFormatException nfe ) {
+ maxRetries = 20;
+ }
+
EntityManager em = getEntityManager(getManagementAppId());
+
+ int retryCount = 0;
+
+ while ( managementApp != null && retryCount++ <= maxRetries ) {
+
+ try {
+ managementApp = em.getApplication();
+
+ if ( managementApp == null ) {
+
+ logger.warn( "Management application not found, attempting creation" );
+
+ Map mgmtAppProps = new HashMap<String, Object>();
+ mgmtAppProps.put( PROPERTY_NAME, CassandraService.MANAGEMENT_APPLICATION );
+ em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps );
+ managementApp = em.getApplication();
+ }
+
+ } catch ( Throwable t ) {
+ logger.warn("Error getting or creating management application after " + retryCount + " retries", t);
+ }
+ }
+
indexService.queueInitializeApplicationIndex(CpNamingUtils.getApplicationScope(getManagementAppId()));
+ if ( managementApp == null ) {
+ throw new RuntimeException("FATAL ERROR: Failed to get or create management app");
+ }
+ }
+
+
+ public Application getManagementApplication() {
+
+ Application ret = null;
+ EntityManager em = getEntityManager(getManagementAppId());
try {
- if ( em.getApplication() == null ) {
- logger.info("Creating management application");
- Map mgmtAppProps = new HashMap<String, Object>();
- mgmtAppProps.put(PROPERTY_NAME, CassandraService.MANAGEMENT_APPLICATION);
- em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps);
- em.getApplication();
- }
+ ret = em.getApplication();
+ managementApp = ret;
- } catch (Exception ex) {
- throw new RuntimeException("Fatal error creating management application", ex);
+ } catch (Exception e) {
+ logger.warn("Error getting management app, returning cached copy version");
}
+
+ return managementApp;
}
@@ -201,9 +242,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache,
- metricsFactory, entityManagerFig, graphManagerFactory, collectionService, connectionService,indexSchemaCacheFactory, applicationId );
-
+ EntityManager em = new CpEntityManager(
+ this,
+ cassandraService,
+ counterUtils,
+ indexService,
+ managerCache,
+ metricsFactory,
+ entityManagerFig,
+ graphManagerFactory,
+ collectionService,
+ connectionService,
+ indexSchemaCacheFactory,
+ applicationId );
return em;
}
@@ -215,7 +266,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public Entity createApplicationV2(
- String orgName, String name, UUID applicationId, Map<String, Object> properties, boolean forMigration) throws Exception {
+ String orgName, String name, UUID applicationId, Map<String, Object> properties, boolean forMigration)
+ throws Exception {
String appName = buildAppName( orgName, name );
@@ -317,8 +369,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// find application_info for application to delete
- migrateAppInfo(applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS, CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking()
- .lastOrDefault( null );
+ migrateAppInfo(
+ applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS,
+ CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking().lastOrDefault( null );
}
//TODO: return status for restore
@@ -360,9 +413,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
* @return
* @throws Exception
*/
- private Observable migrateAppInfo(final UUID applicationUUID, final String deleteTypeName, final String createCollectionName, final String createTypeName ) throws Exception {
+ private Observable migrateAppInfo(
+ final UUID applicationUUID, final String deleteTypeName, final String createCollectionName,
+ final String createTypeName ) throws Exception {
+
+ final ApplicationScope managementAppScope =
+ CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
- final ApplicationScope managementAppScope = CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
final EntityManager managementEm = getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
//the application id we will be removing
@@ -417,7 +474,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
final Id managementAppId = CpNamingUtils.getManagementApplicationId();
final EntityIndex aei = getManagementIndex();
final GraphManager managementGraphManager = managerCache.getGraphManager(managementAppScope);
- final Edge createEdge = CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
+ final Edge createEdge =
+ CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
final Observable createNodeGraph = managementGraphManager.writeEdge(createEdge);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0c0c875/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index ef1baaf..993bf35 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@ -68,9 +68,6 @@ public class ServiceManager {
public static final String APPLICATION_REQUESTS_PER = APPLICATION_REQUESTS + ".";
public static final String IMPL = "Impl";
- public static final String SERVICE_MANAGER_RETRY_INTERVAL = "service.manager.retry.interval";
-
- public static final String SERVICE_MANAGER_MAX_RETRIES= "service.manager.max.retries";
private Application application;
@@ -99,69 +96,27 @@ public class ServiceManager {
this.qm = qm;
this.properties = properties;
- Integer retryInterval;
- try {
- Object retryIntervalObject = properties.get( SERVICE_MANAGER_RETRY_INTERVAL ).toString();
- retryInterval = Integer.parseInt( retryIntervalObject.toString() );
- } catch ( NumberFormatException nfe ) {
- retryInterval = 15000;
- }
-
- Integer maxRetries;
- try {
- Object maxRetriesObject = properties.get( SERVICE_MANAGER_MAX_RETRIES ).toString();
- maxRetries = Integer.parseInt( maxRetriesObject.toString() );
- } catch ( NumberFormatException nfe ) {
- maxRetries = 5;
- }
-
if ( em != null ) {
-
try {
- int retryCount = 0;
- boolean appNotFound = true;
-
- while ( appNotFound && retryCount <= maxRetries ) {
-
- application = em.getApplication();
-
- if ( application != null ) {
- appNotFound = false;
- applicationId = application.getUuid();
-
- } else {
- // Cassandra may be alive but responding very slowly, let's wait and retry
- logger.error("STARTUP PROBLEM: Cannot get application by UUID. Will retry in {} seconds #{}",
- retryInterval/1000, retryCount);
- Thread.sleep( retryInterval );
- retryCount++;
- }
- }
-
- if ( application == null ) {
- Exception e = new RuntimeException(
- "STARTUP FAILURE: application id {" + em.getApplicationId()
- + "} is returning null after " + retryCount + " retries" );
+ application = em.getApplication();
+ if(application == null){
+ Exception e = new RuntimeException("application id {"+em.getApplicationId()+"} is returning null");
+ logger.error("Failed to get application",e);
throw e;
}
-
- } catch ( RuntimeException re ) {
- logger.error( "ServiceManager init failure", re );
- throw re;
-
- } catch ( Exception e ) {
+ applicationId = application.getUuid();
+ }
+ catch ( Exception e ) {
logger.error( "ServiceManager init failure", e );
throw new RuntimeException( e );
}
}
-
if ( properties != null ) {
String packages = properties.getProperty( SERVICE_PACKAGE_PREFIXES );
if ( !StringUtils.isEmpty( packages ) ) {
setServicePackagePrefixes( packages );
}
}
-
return this;
}
[27/44] usergrid git commit: Tests to illustrate case-insensitive
handling of Entity field names.
Posted by sn...@apache.org.
Tests to illustrate case-insensitive handling of Entity field names.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e9228e11
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e9228e11
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e9228e11
Branch: refs/heads/usergrid-1268-akka-211
Commit: e9228e111f338aabf34b334e2661e1966116f23a
Parents: 5452d68
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jun 2 15:33:28 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jun 2 15:33:28 2016 -0400
----------------------------------------------------------------------
.../queries/SelectMappingsQueryTest.java | 79 +++++++++++++-------
1 file changed, 54 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e9228e11/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
index eb6aeee..4a291b6 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.usergrid.rest.applications.queries;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.rest.test.resource.model.Collection;
import org.apache.usergrid.rest.test.resource.model.Entity;
import org.apache.usergrid.rest.test.resource.model.QueryParameters;
@@ -34,6 +35,9 @@ public class SelectMappingsQueryTest extends QueryTestBase {
private static final Logger logger = LoggerFactory.getLogger(OrderByTest.class);
+ /**
+ * Select field mappings may include nested entity fields.
+ */
@Test
public void testNestedSelectFieldNames() throws Exception {
@@ -64,42 +68,67 @@ public class SelectMappingsQueryTest extends QueryTestBase {
/**
- * Shows that field names are case-insensitive.
- * If you define two fields with same name but different cases, behavior is undefined.
+ * When entity posted with two duplicate names with different cases, last one wins.
*/
@Test
- public void testFieldNameCaseSensitivity() throws Exception {
+ public void testMixedCaseDupField() throws Exception {
- int numberOfEntities = 10;
String collectionName = "things";
- Entity[] entities = new Entity[numberOfEntities];
- Entity props = new Entity();
+ String value = RandomStringUtils.randomAlphabetic( 20 );
+ String otherValue = RandomStringUtils.randomAlphabetic( 20 );
- for (int i = 0; i < numberOfEntities; i++) {
- props.put("testProp", "a");
- props.put("testprop", "b");
- entities[i] = app().collection(collectionName).post(props);
- }
+ // create entity with testProp=value
+ Entity entity = new Entity()
+ .withProp( "testProp", value )
+ .withProp( "TESTPROP", otherValue);
+ app().collection( collectionName ).post( entity );
refreshIndex();
- {
- QueryParameters params = new QueryParameters()
- .setQuery( "select * where testProp = 'b'" );
- Collection things = this.app().collection( "things" ).get( params );
+ // testProp and TESTPROP should now have otherValue
- // if field names were case sensitive, this would fail
- assertEquals( numberOfEntities, things.getNumOfEntities() );
- }
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testProp='" + otherValue + "'" );
+ Collection things = this.app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+
+ params = new QueryParameters()
+ .setQuery( "select * where TESTPROP='" + otherValue + "'" );
+ things = app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+ }
- {
- QueryParameters params = new QueryParameters()
- .setQuery( "select * where testprop='b'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( numberOfEntities, things.getNumOfEntities() );
- }
+ /**
+ * Field named testProp can be over-written by field named TESTPROP.
+ */
+ @Test
+ public void testFieldOverride() throws Exception {
- }
+ String collectionName = "things";
+ // create entity with testProp=value
+ String value = RandomStringUtils.randomAlphabetic( 20 );
+ Entity entity = new Entity().withProp( "testProp", value );
+ app().collection( collectionName ).post( entity );
+ refreshIndex();
+
+ // override with TESTPROP=newValue
+ String newValue = RandomStringUtils.randomAlphabetic( 20 );
+ entity = new Entity().withProp( "TESTPROP", newValue );
+ app().collection( collectionName ).post( entity );
+ refreshIndex();
+
+ // testProp and TESTPROP should now have newValue
+
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testProp='" + newValue + "'" );
+ Collection things = this.app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+
+ params = new QueryParameters()
+ .setQuery( "select * where TESTPROP='" + newValue + "'" );
+ things = app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+ }
}
[25/44] usergrid git commit: Merge branch 'release-2.1.1' into
select-mappings-nested
Posted by sn...@apache.org.
Merge branch 'release-2.1.1' into select-mappings-nested
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/151abf78
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/151abf78
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/151abf78
Branch: refs/heads/usergrid-1268-akka-211
Commit: 151abf78f92cebd2ac8701d18b9754f65b88cfdc
Parents: e9e7c39 0970e1d
Author: Dave Johnson <sn...@apache.org>
Authored: Tue May 31 13:32:42 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue May 31 13:32:42 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 5 --
.../applications/ApplicationResourceIT.java | 5 +-
.../usergrid/security/shiro/ShiroCache.java | 44 +++++++++++++-
.../usergrid/services/ServiceManager.java | 61 +++-----------------
4 files changed, 52 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
[32/44] usergrid git commit: Merge commit 'refs/pull/518/head' of
github.com:apache/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge commit 'refs/pull/518/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e6cf6ed4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e6cf6ed4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e6cf6ed4
Branch: refs/heads/usergrid-1268-akka-211
Commit: e6cf6ed4f3bd8b542fc72c3bc1d558898bad88a4
Parents: a1cb1f5 1d0e73b
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jun 3 07:46:06 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jun 3 07:46:06 2016 -0700
----------------------------------------------------------------------
.../usergrid/rest/security/SecuredResourceFilterFactory.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[42/44] usergrid git commit: Merge commit 'refs/pull/517/head' of
github.com:apache/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge commit 'refs/pull/517/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/72c9df14
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/72c9df14
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/72c9df14
Branch: refs/heads/usergrid-1268-akka-211
Commit: 72c9df14a4185a9dafe55255b9d2cda73ad27882
Parents: 0eaea98 9edfda6
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Jun 7 09:15:25 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Jun 7 09:15:25 2016 -0700
----------------------------------------------------------------------
.../persistence/entities/Notification.java | 38 ++++++---
.../collection/CollectionsResourceIT.java | 87 ++++++++++++++++----
.../resource/endpoints/CollectionEndpoint.java | 24 +++---
.../impl/ApplicationQueueManagerImpl.java | 52 +++++++++---
.../AbstractServiceNotificationIT.java | 26 +++---
5 files changed, 158 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
[44/44] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
Posted by sn...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
# Conflicts:
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
# stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
# stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a93cad8f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a93cad8f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a93cad8f
Branch: refs/heads/usergrid-1268-akka-211
Commit: a93cad8fae58679fcbfde9727aa675962f1347d4
Parents: 324ef35 3df0779
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Jun 14 13:14:34 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Jun 14 13:14:34 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
.../corepersistence/CpEntityManager.java | 2 +-
.../corepersistence/CpEntityManagerFactory.java | 244 +++++++++++++++----
.../corepersistence/EntityManagerFig.java | 2 +-
.../asyncevents/AsyncEventService.java | 6 +-
.../asyncevents/AsyncEventServiceImpl.java | 11 +-
.../asyncevents/EventBuilder.java | 6 +-
.../asyncevents/EventBuilderImpl.java | 46 ++--
.../model/DeIndexOldVersionsEvent.java | 12 +-
.../corepersistence/index/IndexService.java | 23 +-
.../corepersistence/index/IndexServiceImpl.java | 74 +++---
.../pipeline/cursor/CursorSerializerUtil.java | 9 -
.../pipeline/cursor/RequestCursor.java | 29 ++-
.../read/search/CandidateEntityFilter.java | 94 ++++++-
.../apache/usergrid/locking/LockManager.java | 5 +
.../locking/cassandra/AstyanaxLockImpl.java | 2 +-
.../cassandra/AstyanaxLockManagerImpl.java | 113 +++++----
.../locking/noop/NoOpLockManagerImpl.java | 5 +
.../persistence/entities/Notification.java | 38 ++-
.../usergrid/persistence/CoreSchemaManager.java | 8 +-
.../apache/usergrid/persistence/IndexIT.java | 98 ++++++++
.../test/resources/usergrid-test-context.xml | 1 +
.../collection/EntityCollectionManager.java | 10 +-
.../exception/CollectionRuntimeException.java | 11 +
.../impl/EntityCollectionManagerImpl.java | 40 ++-
.../serialization/SerializationFig.java | 5 +
.../serialization/impl/LogEntryIterator.java | 128 ++++++++++
.../core/astyanax/CassandraCluster.java | 24 +-
.../persistence/core/astyanax/CassandraFig.java | 18 +-
.../model/field/value/EntityObject.java | 15 +-
.../usergrid/persistence/index/EntityIndex.java | 5 +-
.../usergrid/persistence/index/IndexFig.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 64 ++---
.../persistence/index/impl/EntityIndexTest.java | 41 ----
.../security/SecuredResourceFilterFactory.java | 2 +-
.../collection/CollectionsResourceIT.java | 47 ++++
.../queries/SelectMappingsQueryTest.java | 168 +++++++++++++
.../resource/endpoints/CollectionEndpoint.java | 24 +-
.../test/resources/usergrid-test-context.xml | 2 +-
.../usergrid/security/shiro/ShiroCache.java | 80 +++---
.../usergrid/services/ServiceManager.java | 8 +
.../services/ServiceManagerFactory.java | 12 +
.../impl/ApplicationQueueManagerImpl.java | 52 +++-
.../AbstractServiceNotificationIT.java | 26 +-
.../test/resources/usergrid-test-context.xml | 1 +
45 files changed, 1232 insertions(+), 384 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 926cc0a,18df205..090400f
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -16,24 -16,17 +16,17 @@@
package org.apache.usergrid.corepersistence;
- import java.util.HashMap;
- import java.util.Map;
- import java.util.Set;
- import java.util.TreeMap;
- import java.util.UUID;
-
- import org.apache.usergrid.persistence.collection.uniquevalues.AkkaFig;
- import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import org.springframework.beans.BeansException;
- import org.springframework.context.ApplicationContext;
- import org.springframework.context.ApplicationContextAware;
-
+ import com.google.common.base.Optional;
+ import com.google.common.cache.CacheBuilder;
+ import com.google.common.cache.CacheLoader;
+ import com.google.common.cache.LoadingCache;
+ import com.google.common.util.concurrent.UncheckedExecutionException;
+ import com.google.inject.Injector;
+ import com.google.inject.Key;
+ import com.google.inject.TypeLiteral;
import org.apache.commons.lang.StringUtils;
-
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
-import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
+import org.apache.usergrid.corepersistence.index.CollectionSettingsCacheFactory;
import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.corepersistence.service.CollectionService;
@@@ -53,7 -39,8 +39,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+ import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
++import org.apache.usergrid.persistence.collection.uniquevalues.AkkaFig;
++import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@@ -104,16 -84,17 +86,16 @@@ public class CpEntityManagerFactory imp
private Setup setup = null;
+ EntityManager managementAppEntityManager = null;
+
// cache of already instantiated entity managers
- private LoadingCache<UUID, EntityManager> entityManagers
- = CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() {
- public EntityManager load(UUID appId) { // no checked exception
- return _getEntityManager(appId);
- }
- });
+ private final String ENTITY_MANAGER_CACHE_SIZE = "entity.manager.cache.size";
+ private final LoadingCache<UUID, EntityManager> entityManagers;
private final ApplicationIdCache applicationIdCache;
- //private final IndexSchemaCache indexSchemaCache;
+ Application managementApp = null;
+
private ManagerCache managerCache;
private CassandraService cassandraService;
@@@ -125,11 -106,15 +107,16 @@@
private final CollectionService collectionService;
private final ConnectionService connectionService;
private final GraphManagerFactory graphManagerFactory;
- private final IndexSchemaCacheFactory indexSchemaCacheFactory;
+ private final CollectionSettingsCacheFactory collectionSettingsCacheFactory;
+ private UniqueValuesService uniqueValuesService;
+ private final LockManager lockManager;
- public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
- final Injector injector ) {
+ public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries";
+ public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval";
+
+
+ public CpEntityManagerFactory(
+ final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector ) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
@@@ -142,24 -127,162 +129,175 @@@
this.graphManagerFactory = injector.getInstance( GraphManagerFactory.class );
this.collectionService = injector.getInstance( CollectionService.class );
this.connectionService = injector.getInstance( ConnectionService.class );
- this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
+
+ this.collectionSettingsCacheFactory = injector.getInstance( CollectionSettingsCacheFactory.class );
+
+ AkkaFig akkaFig = injector.getInstance( AkkaFig.class );
+
+ // this line always needs to be last due to the temporary cicular dependency until spring is removed
+ this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
+ getManagementEntityManager() );
+
+ if ( akkaFig.getAkkaEnabled() ) {
+ try {
+ this.uniqueValuesService = injector.getInstance( UniqueValuesService.class );
+ this.uniqueValuesService.start();
+ } catch (Throwable t) {
+ logger.error("Error starting Akka", t);
+ throw t;
+ }
+ }
+ this.lockManager = injector.getInstance( LockManager.class );
+
+ Properties properties = cassandraService.getProperties();
+
+ entityManagers = createEntityManagerCache( properties );
+
+ checkManagementApp( properties );
-
- // this line always needs to be last due to the temporary circular dependency until spring is removed
- applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class)
- .getInstance( getManagementEntityManager() );
+ }
+
+
+ private LoadingCache<UUID, EntityManager> createEntityManagerCache(Properties properties) {
+
+ int entityManagerCacheSize = 100;
+ try {
+ entityManagerCacheSize = Integer.parseInt( properties.getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + ". Will use " + entityManagerCacheSize, e );
+ }
+
+ return CacheBuilder.newBuilder()
+ .maximumSize(entityManagerCacheSize)
+ .build(new CacheLoader<UUID, EntityManager>() {
+
+ public EntityManager load( UUID appId ) { // no checked exception
+
+ // create new entity manager and pre-fetch its application
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Throwable throwable = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Throwable t) {
+ throwable = t;
+ }
+
+ // the management app is a special case
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+
+ if ( app != null ) {
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
+
+ } else if ( managementAppEntityManager != null ) {
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
+ logger.error("Failed to fetch management app");
+ }
+ }
+
+ // missing keyspace means we have not done bootstrap yet
+ final boolean isBootstrapping;
+ if ( throwable instanceof CollectionRuntimeException ) {
+ CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
+ isBootstrapping = cre.isBootstrapping();
+ } else {
+ isBootstrapping = false;
+ }
+
+ // work around for https://issues.apache.org/jira/browse/USERGRID-1291
+ // throw exception so that we do not cache
+ // TODO: determine how application name can intermittently be null
+ if ( app != null && app.getName() == null ) {
+ throw new RuntimeException( "Name is null for application " + appId, throwable );
+ }
+
+ if ( app == null && !isBootstrapping ) {
+ throw new RuntimeException( "Error getting application " + appId, throwable );
+
+ } // else keyspace is missing because setup/bootstrap not done yet
+
+ return entityManager;
+ }
+ });
+ }
+
+
+ private void checkManagementApp(Properties properties) {
+
+ int maxRetries = 100;
+ try {
+ maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_MAXRETRIES, "100" ));
+
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + MANAGEMENT_APP_INIT_MAXRETRIES + ". Will use " + maxRetries, e );
+ }
+
+ int interval = 1000;
+ try {
+ interval = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_INTERVAL, "1000" ));
+
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + MANAGEMENT_APP_INIT_INTERVAL + ". Will use " + maxRetries, e );
+ }
+
+ // hold up construction until we can access the management app
+ int retries = 0;
+ boolean managementAppFound = false;
+ boolean bootstrapping = false;
+ Set<Class> seenBefore = new HashSet<>(10);
+ while ( !managementAppFound && retries++ < maxRetries ) {
+ try {
+ // bypass entity manager cache and get managementApp
+ managementApp = _getEntityManager( getManagementAppId() ).getApplication();
+ managementAppFound = true;
+
+ } catch ( Throwable t ) {
+
+ if ( t instanceof CollectionRuntimeException ) {
+ CollectionRuntimeException cre = (CollectionRuntimeException)t;
+ if ( cre.isBootstrapping() ) {
+ // we're bootstrapping, ignore this and continue
+ bootstrapping = true;
+ break;
+ }
+ }
+ Throwable cause = t;
+
+ // there was an error, be as informative as possible
+ StringBuilder sb = new StringBuilder();
+ sb.append(retries).append(": Error (");
+
+ if ( t instanceof UncheckedExecutionException ) {
+ UncheckedExecutionException uee = (UncheckedExecutionException)t;
+ if ( uee.getCause() instanceof RuntimeException ) {
+ cause = uee.getCause().getCause();
+ sb.append(cause.getClass().getSimpleName()).append(") ")
+ .append(uee.getCause().getMessage());
+ } else {
+ cause = uee.getCause();
+ sb.append(cause.getClass().getSimpleName()).append(") ").append(t.getMessage());
+ }
+ } else {
+ sb.append(t.getCause().getClass().getSimpleName()).append(") ").append(t.getMessage());
+ }
+
+ String msg = sb.toString();
+ if ( !seenBefore.contains( cause.getClass() ) ) {
+ logger.error( msg, t);
+ } else {
+ logger.error(msg);
+ }
+ seenBefore.add( cause.getClass() );
+
+ try { Thread.sleep( interval ); } catch (InterruptedException ignored) {}
+ }
+ }
+
+ if ( !managementAppFound && !bootstrapping ) {
+ // exception here will prevent WAR from being deployed
+ throw new RuntimeException( "Unable to get management app after " + retries + " retries" );
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 46c7a1d,655a968..872ffbb
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@@ -39,15 -39,6 +39,15 @@@ public interface EntityManagerFig exten
int sleep();
@Key( "usergrid.entityManager.enable_deindex_on_update" )
- @Default( "false" )
+ @Default( "true" )
boolean getDeindexOnUpdate();
+
+ /**
+ * Comma-separated list of one or more Amazon regions to use if multiregion
+ * is set to true.
+ */
+ @Key( "usergrid.queue.regionList" )
+ @Default("us-east-1")
+ String getRegionList();
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index ab2caa3,54b18bb..d371b8a
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@@ -43,13 -51,14 +43,12 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.utils.InflectionUtils;
- import org.apache.usergrid.utils.UUIDUtils;
-
-import com.codahale.metrics.Timer;
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import rx.Observable;
+import java.util.*;
+
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.*;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 544e7bb,fcf1c6b..09bd01b
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -27,7 -27,7 +27,8 @@@ import java.util.List
import java.util.UUID;
import com.netflix.astyanax.model.ConsistencyLevel;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
+ import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a93cad8f/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index 48e125b,690cec2..f13ed46
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@@ -17,24 -17,44 +17,26 @@@
package org.apache.usergrid.rest.applications.collection;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-
-import javax.ws.rs.BadRequestException;
-import javax.ws.rs.ClientErrorException;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
+import com.fasterxml.jackson.databind.JsonNode;
import org.apache.commons.lang.RandomStringUtils;
-
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.entities.Application;
+ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.test.resource.AbstractRestIT;
-import org.apache.usergrid.rest.test.resource.model.ApiResponse;
+import org.apache.usergrid.rest.test.resource.model.*;
import org.apache.usergrid.rest.test.resource.model.Collection;
-import org.apache.usergrid.rest.test.resource.model.Credentials;
-import org.apache.usergrid.rest.test.resource.model.Entity;
-import org.apache.usergrid.rest.test.resource.model.QueryParameters;
-import org.apache.usergrid.rest.test.resource.model.Token;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.fasterxml.jackson.databind.JsonNode;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.ClientErrorException;
+import java.io.IOException;
+import java.util.*;
+ import static junit.framework.TestCase.assertNotNull;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
/**
@@@ -1003,58 -1103,48 +1005,103 @@@ public class CollectionsResourceIT exte
assertEquals( 0, connectionsByQuery.getNumOfEntities() );
}
+
+ @Test
+ public void testCollectionRegion() {
+
+ // create collection with settings for index all
+
+ String randomizer = RandomStringUtils.randomAlphanumeric(10);
+ String collectionName = "col_" + randomizer;
+
+ app().collection( collectionName ).collection( "_settings" )
+ .post( new Entity().chainPut( "fields", "all" ) );
+ refreshIndex();
+
+ // get collection settings, should see no region
+
+ Collection collection = app().collection( collectionName ).collection( "_settings" ).get();
+ Map<String, Object> settings = (Map<String, Object>)collection.getResponse().getData();
+ assertNull( settings.get( "region" ));
+
+ // set collection region with bad region, expect error
+
+ try {
+ app().collection( collectionName ).collection( "_settings" )
+ .post( new Entity().chainPut( "region", "us-moon-1" ) );
+ fail( "post should have failed");
+
+ } catch ( BadRequestException expected ) {}
+
+ // set collection region with good region
+
+ app().collection( collectionName ).collection( "_settings" )
+ .post( new Entity().chainPut( "region", "us-east-1" ) );
+
+ // get collection settings see that we have a region
+
+ collection = app().collection( collectionName ).collection( "_settings" ).get();
+ settings = (Map<String, Object>)collection.getResponse().getData();
+ assertNotNull( settings.get( "region" ));
+ assertEquals( "us-east-1", settings.get( "region" ));
+
+ // unset the collection region
+
+ app().collection( collectionName ).collection( "_settings" )
+ .post( new Entity().chainPut( "region", "" ) );
+ refreshIndex();
+
+ // get collection settings, should see no region
+
+ collection = app().collection( collectionName ).collection( "_settings" ).get();
+ settings = (Map<String, Object>)collection.getResponse().getData();
+ assertNull( settings.get( "region" ));
+
+
+ }
++
++
+ @Test
+ public void testBeingAbleToRetreiveMigratedValues() throws Exception {
+
+
+ Entity notifier = new Entity().chainPut("name", "mynotifier").chainPut("provider", "noop");
+
+ ApiResponse notifierNode = this.pathResource(getOrgAppPath("notifier")).post(ApiResponse.class,notifier);
+
+ // create user
+
+ Map payloads = new HashMap<>( );
+ payloads.put( "mynotifier","hello world" );
+
+ Map statistics = new HashMap<>( );
+ statistics.put( "sent",1 );
+ statistics.put( "errors",2 );
+
+ Entity payload = new Entity();
+ payload.put("debug", false);
+ payload.put( "expectedCount",0 );
+ payload.put( "finished",1438279671229L);
+ payload.put( "payloads",payloads);
+ payload.put( "priority","normal");
+ payload.put( "state","FINISHED");
+ payload.put( "statistics",statistics);
+
+
+ this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).post(payload );
+ this.refreshIndex();
+
+ Collection user2 = this.app().collection("notifications").get();
+
+ assertEquals(1,user2.getNumOfEntities());
+
+ this.app().collection("notifications/"+ UUIDUtils.newTimeUUID()).put(null,payload );
+ this.refreshIndex();
+
+ user2 = this.app().collection("notifications").get();
+
+ assertEquals(2,user2.getNumOfEntities());
+
+ }
+
-
}
[07/44] usergrid git commit: Improve cache logic to avoid caching
"bad" entity managers and to ensure that manager app is always available.
Should be no reason to use a retry loop because the system will continue to
call for EntityManagers. Adds new "ent
Posted by sn...@apache.org.
Improve cache logic to avoid caching "bad" entity managers and to ensure that manager app is always available.
Should be no reason to use a retry loop because the system will continue to call for EntityManagers.
Adds new "entity.manager.cache.size" property, which defaults to 100.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5f463412
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5f463412
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5f463412
Branch: refs/heads/usergrid-1268-akka-211
Commit: 5f463412b661c7b681489d8831fc2c7c7d32938a
Parents: 3a96239
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 18 09:18:21 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 18 09:18:21 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 70 ++++++++++++++++++--
1 file changed, 63 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f463412/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 91a936d..4028875 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -102,13 +102,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Setup setup = null;
+ EntityManager managementAppEntityManager = null;
+
// cache of already instantiated entity managers
- private LoadingCache<UUID, EntityManager> entityManagers
- = CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() {
- public EntityManager load(UUID appId) { // no checked exception
- return _getEntityManager(appId);
- }
- });
+ private final String ENTITY_MANAGER_CACHE_SIZE = "entity.manager.cache.size";
+ private final LoadingCache<UUID, EntityManager> entityManagers;
+
+
+
private final ApplicationIdCache applicationIdCache;
//private final IndexSchemaCache indexSchemaCache;
@@ -142,12 +143,67 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
- //this line always needs to be last due to the temporary cicular dependency until spring is removed
+ // this line always needs to be last due to the temporary circular dependency until spring is removed
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
+ int entityManagerCacheSize = 100;
+ try {
+ entityManagerCacheSize = Integer.parseInt(
+ cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e );
+ }
+
+ entityManagers = CacheBuilder.newBuilder()
+ .maximumSize(entityManagerCacheSize)
+ .build(new CacheLoader<UUID, EntityManager>() {
+
+ public EntityManager load( UUID appId ) { // no checked exception
+
+ // get entity manager and ensure it can get its own application
+
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Exception exception = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Exception e) {
+ exception = e;
+ }
+
+ // the management app is a special case
+
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+
+ if ( app != null ) {
+
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
+
+ } else if ( managementAppEntityManager != null ) {
+
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
+
+ } else {
+
+ // fetch failed and we have nothing cached, we must be bootstrapping
+ logger.info("managementAppEntityManager is null, bootstrapping in progress");
+ }
+
+ } else { // not the management app, so blow up if app cannot be fetched
+
+ if (app == null) {
+ throw new RuntimeException( "Error getting application " + appId, exception );
+ }
+ }
+
+ return entityManager;
+ }
+ });
}
[22/44] usergrid git commit: Make retries and interval configurable
Posted by sn...@apache.org.
Make retries and interval configurable
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fcd00e83
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fcd00e83
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fcd00e83
Branch: refs/heads/usergrid-1268-akka-211
Commit: fcd00e83b4febea1f395e7d8818bab8c38fa8bea
Parents: 910811d
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 18:30:27 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 18:30:27 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 8 ++
.../corepersistence/CpEntityManagerFactory.java | 94 ++++++++++++--------
.../cassandra/AstyanaxLockManagerImpl.java | 5 +-
.../persistence/core/astyanax/CassandraFig.java | 18 +++-
4 files changed, 86 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 4f57cdd..1e22d6a 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -195,6 +195,12 @@ cassandra.lock.writecl=LOCAL_QUORUM
#
#cassandra.useSocketKeepalive=false
+# Number of times to retry creation of lock keyspace and column family
+cassandra.lock.init.retries = 100;
+
+# Interval between lock keyspace creation attempts (in milliseconds)
+cassandra.lock.init.interval = 1000;
+
##################### Async Threadpool Settings #####################
@@ -628,6 +634,8 @@ usergrid.auth.cache.inmemory.size=3000
# all (= in + out)'
usergrid.rest.default-connection-param=all
+# Number of times to retry attempt to access management app on startup
+management.app.max.retries=100
############################## Usergrid Testing #############################
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 84872aa..b6fbc2a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -23,6 +23,8 @@ import com.google.common.cache.LoadingCache;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
+import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
@@ -124,54 +126,79 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
+ Properties properties = cassandraService.getProperties();
+
+ entityManagers = createEntityManagerCache( properties );
+
+ checkManagementApp( properties );
+
+ // this line always needs to be last due to the temporary circular dependency until spring is removed
+ applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class)
+ .getInstance( getManagementEntityManager() );
+ }
+
+
+ private LoadingCache<UUID, EntityManager> createEntityManagerCache(Properties properties) {
+
int entityManagerCacheSize = 100;
try {
- entityManagerCacheSize = Integer.parseInt(
- cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ entityManagerCacheSize = Integer.parseInt( properties.getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
} catch ( Exception e ) {
- logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e );
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + ". Will use " + entityManagerCacheSize, e );
}
- entityManagers = CacheBuilder.newBuilder()
+ return CacheBuilder.newBuilder()
.maximumSize(entityManagerCacheSize)
.build(new CacheLoader<UUID, EntityManager>() {
- public EntityManager load( UUID appId ) { // no checked exception
+ public EntityManager load( UUID appId ) { // no checked exception
- // create new entity manager and pre-fetch its application
- EntityManager entityManager = _getEntityManager( appId );
- Application app = null;
- Exception exception = null;
- try {
- app = entityManager.getApplication();
- } catch (Exception e) {
- exception = e;
- }
+ // create new entity manager and pre-fetch its application
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Throwable throwable = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Throwable t) {
+ throwable = t;
+ }
- // the management app is a special case
+ // the management app is a special case
- if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
- if ( app != null ) {
- // we successfully fetched up the management app, cache it for a rainy day
- managementAppEntityManager = entityManager;
+ if ( app != null ) {
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
- } else if ( managementAppEntityManager != null ) {
- // failed to fetch management app, use cached one
- entityManager = managementAppEntityManager;
+ } else if ( managementAppEntityManager != null ) {
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
+ }
+ }
+
+ if ( app == null && throwable != null && throwable.getCause() instanceof BadRequestException) {
+ // probably means schema has not been created yet
+ } else {
+ throw new RuntimeException( "Error getting application " + appId, throwable );
}
- }
- if (app == null) {
- throw new RuntimeException( "Error getting application " + appId, exception );
+ return entityManager;
}
+ });
+ }
- return entityManager;
- }
- });
+
+ private void checkManagementApp(Properties properties) {
+ int maxRetries = 100;
+ try {
+ maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_MAX_RETRIES, "100" ));
+
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + MANAGEMENT_APP_MAX_RETRIES + ". Will use " + maxRetries, e );
+ }
// hold up construction until we can access the management app
- int maxRetries = 1000;
int retries = 0;
boolean managementAppFound = false;
while ( !managementAppFound && retries++ < maxRetries ) {
@@ -186,6 +213,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
} else {
logger.error(msg);
}
+ try { Thread.sleep( 1000 ); } catch (InterruptedException ignored) {}
}
}
@@ -193,10 +221,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// exception here will prevent WAR from being deployed
throw new RuntimeException( "Unable to get management app after " + retries + " retries" );
}
-
- // this line always needs to be last due to the temporary circular dependency until spring is removed
- applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class)
- .getInstance( getManagementEntityManager() );
}
@@ -240,8 +264,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
try {
return entityManagers.get( applicationId );
}
- catch ( Exception ex ) {
- logger.error("Error getting oldAppInfo manager", ex);
+ catch ( Throwable t ) {
+ logger.error("Error getting entity manager", t);
}
return _getEntityManager(applicationId);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 49ff52e..9bf6694 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -52,6 +52,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
private ColumnFamily columnFamily;
private static final int MINIMUM_LOCK_EXPIRATION = 60000; // 1 minute
+
@Inject
public AstyanaxLockManagerImpl(CassandraFig cassandraFig,
CassandraCluster cassandraCluster ) throws ConnectionException {
@@ -59,7 +60,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
this.cassandraFig = cassandraFig;
// hold up construction until we can create the column family
- int maxRetries = 1000;
+ int maxRetries = cassandraFig.getLockManagerInitRetries();
int retries = 0;
boolean famReady = false;
while ( !famReady && retries++ < maxRetries ) {
@@ -76,7 +77,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
} else {
logger.error( msg );
}
- try { Thread.sleep(1000); } catch (InterruptedException ignored) {}
+ try { Thread.sleep( cassandraFig.getLockManagerInitInterval() ); } catch (InterruptedException ignored) {}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
index b00eca8..d315561 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
@@ -50,8 +50,8 @@ public interface CassandraFig extends GuicyFig {
String LOCKS_CONNECTIONS = "cassandra.lock.connections";
String LOCKS_EXPIRATION = "cassandra.lock.expiration.milliseconds";
-
-
+ String LOCK_MANAGER_INIT_RETRIES = "cassandra.lock.init.retries";
+ String LOCK_MANAGER_INIT_INTERVAL = "cassandra.lock.init.interval";
// re-usable default values
String DEFAULT_CONNECTION_POOLSIZE = "15";
@@ -180,4 +180,18 @@ public interface CassandraFig extends GuicyFig {
@Default(DEFAULT_LOCKS_EXPIRATION)
int getLocksExpiration();
+ /**
+ * How many times to attempt lock keyspace and column family creation
+ */
+ @Key( LOCK_MANAGER_INIT_RETRIES )
+ @Default( "100" )
+ int getLockManagerInitRetries();
+
+ /**
+ * Return the expiration that should be used for expiring a lock if it's not released
+ */
+ @Key( LOCK_MANAGER_INIT_INTERVAL )
+ @Default( "1000" )
+ int getLockManagerInitInterval();
+
}
[43/44] usergrid git commit: Additional enhancements to cursor
validation and error messages.
Posted by sn...@apache.org.
Additional enhancements to cursor validation and error messages.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3df07791
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3df07791
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3df07791
Branch: refs/heads/usergrid-1268-akka-211
Commit: 3df07791c394de2e7ec7e1f9559e1490f63af630
Parents: 72c9df1
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Jun 7 09:32:15 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Jun 7 09:32:15 2016 -0700
----------------------------------------------------------------------
.../pipeline/cursor/CursorSerializerUtil.java | 9 ---------
.../pipeline/cursor/RequestCursor.java | 21 ++++++++++++++++----
2 files changed, 17 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3df07791/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java
index 7acdd00..5728433 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializerUtil.java
@@ -40,12 +40,6 @@ public class CursorSerializerUtil {
private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
- /**
- * Aritrary number, just meant to keep us from having a DOS issue
- */
- private static final int MAX_SIZE = 1024;
-
-
public static ObjectMapper getMapper() {
return MAPPER;
}
@@ -75,9 +69,6 @@ public class CursorSerializerUtil {
*/
public static JsonNode fromString( final String base64EncodedJson ) {
- Preconditions.checkArgument( base64EncodedJson.length() <= MAX_SIZE,
- "Your cursor must be less than " + MAX_SIZE + " chars in length" );
-
final byte[] data = Base64.getUrlDecoder().decode( base64EncodedJson );
JsonNode jsonNode;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3df07791/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
index acd6e25..0209794 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
@@ -35,6 +35,10 @@ import java.util.Map;
*/
public class RequestCursor {
+ /**
+ * Arbitrary number, just meant to keep us from having a DOS issue
+ */
+ private static final int MAX_SIZE = 1024;
private static final int MAX_CURSOR_COUNT = 100;
@@ -74,9 +78,15 @@ public class RequestCursor {
* Deserialize from the cursor as json nodes
*/
private Map<Integer, JsonNode> fromCursor( final String cursor ) throws CursorParseException {
- if(cursor.isEmpty()){
- throw new IllegalArgumentException("cursor cannot be empty");
- }
+
+
+ Preconditions.checkArgument( cursor != null, "Cursor cannot be null");
+
+ Preconditions.checkArgument( cursor.length() <= MAX_SIZE,
+ "Your cursor must be less than " + MAX_SIZE + " chars in length" );
+
+ Preconditions.checkArgument( !cursor.isEmpty(), "Cursor cannot have an empty value");
+
try {
JsonNode jsonNode = CursorSerializerUtil.fromString( cursor );
@@ -95,8 +105,11 @@ public class RequestCursor {
return cursors;
}
+ catch ( IllegalArgumentException ie ){
+ throw new IllegalArgumentException("Provided cursor has an invalid format and cannot be parsed.");
+ }
catch ( Exception e ) {
- throw new CursorParseException( "Unable to serialize cursor", e );
+ throw new CursorParseException( "Unable to deserialize cursor", e );
}
}
}
[21/44] usergrid git commit: Add retry logic and logging in
AstyanaxLockManagerImpl and CpEntityManagerFactory,
also ASL header in CassandraCluster
Posted by sn...@apache.org.
Add retry logic and logging in AstyanaxLockManagerImpl and CpEntityManagerFactory, also ASL header in CassandraCluster
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/910811d2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/910811d2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/910811d2
Branch: refs/heads/usergrid-1268-akka-211
Commit: 910811d258b359a3c207c1ef34014d1ba28afebe
Parents: ba10e7f
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 13:56:51 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 13:56:51 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 8 ++---
.../cassandra/AstyanaxLockManagerImpl.java | 36 +++++++++++++-------
.../core/astyanax/CassandraCluster.java | 24 ++++++++++---
3 files changed, 47 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/910811d2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index d2417be..84872aa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -174,17 +174,17 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
int maxRetries = 1000;
int retries = 0;
boolean managementAppFound = false;
- Set<Class> seenBefore = new HashSet<>(100);
while ( !managementAppFound && retries++ < maxRetries ) {
try {
getEntityManager( getManagementAppId() ).getApplication();
managementAppFound = true;
} catch ( Throwable t ) {
- if ( seenBefore.contains( t.getClass() )) { // don't log full stack trace if we've seen same before
- logger.error("Error {} getting management app on try {}", t.getClass().getSimpleName(), retries);
+ String msg = "Error " + t.getClass() + " getting management app on try " + retries;
+ if ( logger.isDebugEnabled() ) {
+ logger.error( msg, t);
} else {
- logger.error("Error getting management app on try {}", t.getClass().getSimpleName(), t);
+ logger.error(msg);
}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/910811d2/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index a69aee2..49ff52e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -37,9 +37,7 @@ import org.apache.usergrid.persistence.core.astyanax.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.TimeUnit;
@Singleton
@@ -50,8 +48,8 @@ public class AstyanaxLockManagerImpl implements LockManager {
private final CassandraFig cassandraFig;
- private final Keyspace keyspace;
- private final ColumnFamily columnFamily;
+ private Keyspace keyspace;
+ private ColumnFamily columnFamily;
private static final int MINIMUM_LOCK_EXPIRATION = 60000; // 1 minute
@Inject
@@ -59,14 +57,28 @@ public class AstyanaxLockManagerImpl implements LockManager {
CassandraCluster cassandraCluster ) throws ConnectionException {
this.cassandraFig = cassandraFig;
- this.keyspace = cassandraCluster.getLocksKeyspace();
-
- createLocksKeyspace();
-
- this.columnFamily = createLocksColumnFamily();
-
-
+ // hold up construction until we can create the column family
+ int maxRetries = 1000;
+ int retries = 0;
+ boolean famReady = false;
+ while ( !famReady && retries++ < maxRetries ) {
+ try {
+ keyspace = cassandraCluster.getLocksKeyspace();
+ createLocksKeyspace();
+ columnFamily = createLocksColumnFamily();
+ famReady = true;
+
+ } catch ( Throwable t ) {
+ String msg = "Error " + t.getClass().getSimpleName() + " creating locks keyspace try " + retries;
+ if ( logger.isDebugEnabled() ) {
+ logger.error( msg, t );
+ } else {
+ logger.error( msg );
+ }
+ try { Thread.sleep(1000); } catch (InterruptedException ignored) {}
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/910811d2/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java
index 0adac8e..6abc143 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java
@@ -1,15 +1,29 @@
-package org.apache.usergrid.persistence.core.astyanax;
-
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.core.astyanax;
import com.netflix.astyanax.Keyspace;
-
import java.util.Map;
-
public interface CassandraCluster {
-
Map<String, Keyspace> getKeyspaces();
Keyspace getApplicationKeyspace();
[30/44] usergrid git commit: Merge commit 'refs/pull/528/head' of
github.com:apache/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge commit 'refs/pull/528/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8976b414
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8976b414
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8976b414
Branch: refs/heads/usergrid-1268-akka-211
Commit: 8976b414a7e62fa22f9e894eaf50edfe60c4dd76
Parents: 317714b 4b8e546
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jun 3 07:42:30 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jun 3 07:42:30 2016 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
.../corepersistence/CpEntityManagerFactory.java | 243 ++++++++++++++-----
.../apache/usergrid/locking/LockManager.java | 5 +
.../cassandra/AstyanaxLockManagerImpl.java | 30 ++-
.../locking/noop/NoOpLockManagerImpl.java | 5 +
.../exception/CollectionRuntimeException.java | 11 +
.../core/astyanax/CassandraCluster.java | 24 +-
.../persistence/core/astyanax/CassandraFig.java | 18 +-
.../usergrid/services/ServiceManager.java | 8 +
.../services/ServiceManagerFactory.java | 12 +
10 files changed, 285 insertions(+), 74 deletions(-)
----------------------------------------------------------------------