You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/20 15:46:49 UTC
[21/50] [abbrv] git commit: Added verification to init to catch NPE
issues
Added verification to init to catch NPE issues
Updated index creation to write a document and delete to verify correct functionality
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/722c43b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/722c43b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/722c43b7
Branch: refs/heads/two-dot-o-events
Commit: 722c43b731a80c810916a09af8e50c19ac7c6891
Parents: 83200bf
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 16 20:53:59 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 16 20:53:59 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 20 +--
.../usergrid/persistence/EntityManager.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 144 ++++++++++++++-----
3 files changed, 121 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 2498dda..751b9e2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -16,6 +16,7 @@
package org.apache.usergrid.corepersistence;
+import com.google.common.base.Preconditions;
import com.netflix.hystrix.exception.HystrixRuntimeException;
import com.yammer.metrics.annotation.Metered;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
@@ -191,6 +192,9 @@ public class CpEntityManager implements EntityManager {
@Override
public void init( EntityManagerFactory emf, UUID applicationId ) {
+ Preconditions.checkNotNull(emf, "emf must not be null");
+ Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
+
this.emf = ( CpEntityManagerFactory ) emf;
this.managerCache = this.emf.getManagerCache();
this.applicationId = applicationId;
@@ -202,9 +206,6 @@ public class CpEntityManager implements EntityManager {
// set to false for now
this.skipAggregateCounters = false;
-
-
- applicationScope = this.emf.getApplicationScope( applicationId );
}
@@ -643,11 +644,11 @@ public class CpEntityManager implements EntityManager {
return getRelationManager( entityRef ).searchCollection( collectionName, query );
}
-
- @Override
- public void setApplicationId( UUID applicationId ) {
- this.applicationId = applicationId;
- }
+//
+// @Override
+// public void setApplicationId( UUID applicationId ) {
+// this.applicationId = applicationId;
+// }
@Override
@@ -2884,7 +2885,8 @@ public class CpEntityManager implements EntityManager {
}
catch(WriteOptimisticVerifyException wo ){
//swallow this, it just means this was already updated, which accomplishes our task. Just ignore.
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(), collName );
+ logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(),
+ collName );
}
catch (Exception ex) {
logger.error("Error repersisting entity", ex);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 3684d7e..cd92729 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -41,7 +41,7 @@ import org.apache.usergrid.persistence.index.query.Query.Level;
*/
public interface EntityManager {
- public void setApplicationId( UUID applicationId );
+// public void setApplicationId( UUID applicationId );
public GeoIndexManager getGeoIndexManager();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/722c43b7/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index b562d8a..7584386 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -26,7 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
@@ -36,7 +35,9 @@ import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.search.SearchHit;
@@ -58,7 +59,9 @@ import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -88,11 +91,18 @@ public class EsEntityIndexImpl implements EntityIndex {
private final IndexFig config;
- //number of times to wait for the index to refresh propertly. Is an N+1, so 9 = 10
- private static final int MAX_WAITS = 9;
+ //number of times to wait for the index to refresh properly.
+ private static final int MAX_WAITS = 10;
//number of milliseconds to try again before sleeping
private static final int WAIT_TIME = 250;
+ private static final String VERIFY_TYPE = "verification";
+
+ private static final ImmutableMap<String, Object> DEFAULT_PAYLOAD =
+ ImmutableMap.<String, Object>of( "field", "test" );
+
+ private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
+
@Inject
public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
@@ -120,21 +130,16 @@ public class EsEntityIndexImpl implements EntityIndex {
CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).execute().actionGet();
log.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
- RefreshResponse response;
-
- do {
- response = admin.indices().prepareRefresh( indexName ).execute().actionGet();
- }
- while ( response.getFailedShards() != 0 );
+ //create the document, this ensures the index is ready
/**
- * Immediately refresh to ensure the entire cluster is ready to receive this write. Occasionally we see
+ * Immediately create a document and remove it to ensure the entire cluster is ready to receive documents
+ * . Occasionally we see
* errors. See this post.
- * http://elasticsearch-users.115913.n3.nabble
- * .com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html
+ * http://elasticsearch-users.115913.n3.nabble.com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html
*
*/
- refresh();
+ testNewIndex();
}
catch ( IndexAlreadyExistsException expected ) {
// this is expected to happen if index already exists, it's a no-op and swallow
@@ -146,6 +151,44 @@ public class EsEntityIndexImpl implements EntityIndex {
/**
+ * Tests writing a document to a new index to ensure it's working correctly. Comes from email
+ *
+ * http://elasticsearch-users.115913.n3.nabble
+ * .com/IndexMissingException-on-create-index-followed-by-refresh-td1832793.html
+ */
+
+ private void testNewIndex() {
+
+
+ log.info( "Refreshing Created new Index Name [{}]", indexName );
+
+ final RetryOperation retryOperation = new RetryOperation() {
+ @Override
+ public boolean doOp() {
+ final String tempId = UUIDGenerator.newTimeUUID().toString();
+
+
+ client.prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD ).get();
+
+ log.info( "Successfully created new document with docId {} in index {} and type {}", tempId, indexName,
+ VERIFY_TYPE );
+
+ //delete all types, this way if we miss one it will get cleaned up
+
+ client.prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE ).setQuery( MATCH_ALL_QUERY_BUILDER )
+ .get();
+
+ log.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
+
+ return true;
+ }
+ };
+
+ doInRetry( retryOperation );
+ }
+
+
+ /**
* Setup ElasticSearch type mappings as a template that applies to all new indexes. Applies to all indexes that
* start with our prefix.
*/
@@ -277,30 +320,22 @@ public class EsEntityIndexImpl implements EntityIndex {
log.info( "Refreshing Created new Index Name [{}]", indexName );
- //now try to refresh, to ensure that it's recognized by everyone. Occasionally we can get a success
- //before we can write.
- for ( int i = 0; i < MAX_WAITS; i++ ) {
- try {
- client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
- log.debug( "Refreshed index: " + indexName );
- return;
- }
- catch ( IndexMissingException e ) {
- log.error( "Unable to refresh index after create. Waiting before sleeping.", e );
- }
-
- try {
- Thread.sleep( WAIT_TIME );
- }
- catch ( InterruptedException e ) {
- //swallow it
+ final RetryOperation retryOperation = new RetryOperation() {
+ @Override
+ public boolean doOp() {
+ try {
+ client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+ log.debug( "Refreshed index: " + indexName );
+ return true;
+ }
+ catch ( IndexMissingException e ) {
+ log.error( "Unable to refresh index after create. Waiting before sleeping.", e );
+ throw e;
+ }
}
- }
+ };
- /**
- * Try the refresh one last time if we get here
- */
- client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
+ doInRetry( retryOperation );
log.debug( "Refreshed index: " + indexName );
}
@@ -328,4 +363,43 @@ public class EsEntityIndexImpl implements EntityIndex {
log.info( "Failed to delete index " + indexName );
}
}
+
+
+ /**
+ * Do the retry operation
+ * @param operation
+ */
+ private void doInRetry( final RetryOperation operation ) {
+ for ( int i = 0; i < MAX_WAITS; i++ ) {
+
+ try {
+ if(operation.doOp()){
+ return;
+ }
+ }
+ catch ( Exception e ) {
+ log.error( "Unable to execute operation, retrying", e );
+ }
+
+
+ try {
+ Thread.sleep( WAIT_TIME );
+ }
+ catch ( InterruptedException e ) {
+ //swallow it
+ }
+ }
+ }
+
+
+ /**
+ * Interface for operations
+ */
+ private static interface RetryOperation {
+
+ /**
+ * Return true if done, false if there should be a retry
+ */
+ public boolean doOp();
+ }
}