You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/09 20:38:06 UTC
git commit: A optimization: set our ElasticSearch field mappings once
at the cluster level,
so we do not have to set them for every new type that is created.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o 81d4e0ea2 -> c0edbe191
A optimization: set our ElasticSearch field mappings once at the cluster level, so we do not have to set them for every new type that is created.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c0edbe19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c0edbe19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c0edbe19
Branch: refs/heads/two-dot-o
Commit: c0edbe1914b1228dba6c7b06d2da7590def6da13
Parents: 81d4e0e
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 9 14:29:48 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 9 14:29:48 2014 -0400
----------------------------------------------------------------------
.../index/impl/EsEntityIndexBatchImpl.java | 117 ++++++++-----------
.../index/impl/EsEntityIndexImpl.java | 117 ++++++-------------
.../persistence/index/impl/IndexingUtils.java | 66 +++++++++++
3 files changed, 145 insertions(+), 155 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0edbe19/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b88be98..9e70a00 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -18,7 +18,6 @@ package org.apache.usergrid.persistence.index.impl;/*
*/
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -30,10 +29,7 @@ import java.util.UUID;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,6 +58,7 @@ import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
import com.google.common.base.Joiner;
+import java.io.IOException;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
@@ -72,7 +69,10 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PR
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createCollectionScopeTypeName;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.indices.IndexAlreadyExistsException;
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -98,8 +98,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private int count;
- public EsEntityIndexBatchImpl( final ApplicationScope applicationScope, final Client client, final IndexFig config,
- final Set<String> knownTypes, final int autoFlushSize ) {
+ public EsEntityIndexBatchImpl(
+ final ApplicationScope applicationScope, final Client client, final IndexFig config,
+ final Set<String> knownTypes, final int autoFlushSize ) {
this.applicationScope = applicationScope;
this.client = client;
@@ -120,20 +121,23 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
final String indexType = createCollectionScopeTypeName( indexScope );
if ( log.isDebugEnabled() ) {
- log.debug( "Indexing entity {}:{} in scope\n app {}\n owner {}\n name {}\n type {}", new Object[] {
- entity.getId().getType(), entity.getId().getUuid(), applicationScope.getApplication(),
- indexScope.getOwner(), indexScope.getName(), indexType
- } );
+ log.debug( "Indexing entity {}:{} in scope\n app {}\n "
+ + "owner {}\n name {}\n type {}", new Object[] {
+ entity.getId().getType(),
+ entity.getId().getUuid(),
+ applicationScope.getApplication(),
+ indexScope.getOwner(),
+ indexScope.getName(), indexType
+ });
}
ValidationUtils.verifyEntityWrite( entity );
- initType( indexScope, indexType );
-
Map<String, Object> entityAsMap = entityToMap( entity );
// need prefix here becuase we index UUIDs as strings
- entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase() );
+ entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME,
+ entity.getId().getUuid().toString().toLowerCase() );
// let caller add these fields if needed
// entityAsMap.put("created", entity.getId().getUuid().timestamp();
@@ -143,7 +147,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "Indexing entity id {} data {} ", indexId, entityAsMap );
- bulkRequest.add( client.prepareIndex( indexName, indexType, indexId ).setSource( entityAsMap ) );
+ bulkRequest.add(client.prepareIndex( indexName, indexType, indexId).setSource(entityAsMap));
maybeFlush();
@@ -159,16 +163,20 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
final String indexType = createCollectionScopeTypeName( indexScope );
if ( log.isDebugEnabled() ) {
- log.debug( "De-indexing entity {}:{} in scope\n app {}\n owner {}\n name {} type {}", new Object[] {
- id.getType(), id.getUuid(), applicationScope.getApplication(), indexScope.getOwner(),
- indexScope.getName(), indexType
+ log.debug( "De-indexing entity {}:{} in scope\n app {}\n owner {}\n "
+ + "name {} type {}", new Object[] {
+ id.getType(),
+ id.getUuid(),
+ applicationScope.getApplication(),
+ indexScope.getOwner(),
+ indexScope.getName(),
+ indexType
} );
}
String indexId = createIndexDocId( id, version );
- bulkRequest.add( client.prepareDelete( indexName, indexType, indexId ).setRefresh( refresh ) );
-
+ bulkRequest.add( client.prepareDelete( indexName, indexType, indexId ).setRefresh(refresh));
log.debug( "Deindexed Entity with index id " + indexId );
@@ -212,8 +220,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
for ( BulkItemResponse response : responses ) {
if ( response.isFailed() ) {
- throw new RuntimeException(
- "Unable to index documents. Errors are :" + response.getFailure().getMessage() );
+ throw new RuntimeException("Unable to index documents. Errors are :"
+ + response.getFailure().getMessage() );
}
}
@@ -238,48 +246,13 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
/**
- * Create ElasticSearch mapping for each type of Entity.
- */
- private void initType( final IndexScope indexScope, String typeName ) {
-
- // no need for synchronization here, it's OK if we init attempt to init type multiple times
- if ( knownTypes.contains( typeName ) ) {
- return;
- }
-
- AdminClient admin = client.admin();
- try {
- XContentBuilder mxcb = EsEntityIndexImpl.createDoubleStringIndexMapping( jsonBuilder(), typeName );
-
-
- //TODO Dave can this be collapsed into the build as well?
- admin.indices().preparePutMapping( indexName ).setType( typeName ).setSource( mxcb ).execute().actionGet();
-
- admin.indices().prepareGetMappings( indexName ).addTypes( typeName ).execute().actionGet();
-
- // log.debug("Created new type mapping");
- // log.debug(" Scope application: " + indexScope.getApplication());
- // log.debug(" Scope owner: " + indexScope.getOwner());
- // log.debug(" Type name: " + typeName);
-
- knownTypes.add( typeName );
- }
- catch ( IndexAlreadyExistsException ignored ) {
- // expected
- }
- catch ( IOException ex ) {
- throw new RuntimeException(
- "Exception initializing type " + typeName + " in app " + applicationScope.getApplication()
- .toString() );
- }
- }
-
-
- /**
- * Convert Entity to Map. Adding prefixes for types:
- *
- * su_ - String unanalyzed field sa_ - String analyzed field go_ - Location field nu_ - Number field bu_ - Boolean
- * field
+ * Convert Entity to Map and Adding prefixes for types:
+ * <pre>
+ * su_ - String unanalyzed field
+ * sa_ - String analyzed field
+ * go_ - Location field nu_ - Number field
+ * bu_ - Boolean field
+ * </pre>
*/
private static Map entityToMap( EntityObject entity ) {
@@ -291,7 +264,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) );
+ entityMap.put( field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
if ( list.get( 0 ) instanceof String ) {
@@ -304,18 +278,17 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) );
+ entityMap.put( field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof SetField ) {
Set set = ( Set ) field.getValue();
- entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( set ) ) );
+ entityMap.put( field.getName().toLowerCase(),
+ new ArrayList( processCollectionForMap( set ) ) );
}
else if ( f instanceof EntityObjectField ) {
EntityObject eo = ( EntityObject ) field.getValue();
entityMap.put( field.getName().toLowerCase(), entityToMap( eo ) ); // recursion
-
- // Add type information as field-name prefixes
-
}
else if ( f instanceof StringField ) {
@@ -334,7 +307,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
locMap.put( "lon", locField.getValue().getLongitude() );
entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
}
- else if ( f instanceof DoubleField || f instanceof FloatField || f instanceof IntegerField
+ else if ( f instanceof DoubleField
+ || f instanceof FloatField
+ || f instanceof IntegerField
|| f instanceof LongField ) {
entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0edbe19/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 23717b1..8c07d6d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -17,7 +17,6 @@
*/
package org.apache.usergrid.persistence.index.impl;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
@@ -57,16 +56,15 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
+import java.io.IOException;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOC_ID_SEPARATOR_SPLITTER;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createCollectionScopeTypeName;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
+import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.common.xcontent.XContentFactory;
/**
@@ -91,20 +89,18 @@ public class EsEntityIndexImpl implements EntityIndex {
private final IndexFig config;
@Inject
- public EsEntityIndexImpl(@Assisted final ApplicationScope applicationScope,
+ public EsEntityIndexImpl(@Assisted final ApplicationScope appScope,
final IndexFig config, final EsProvider provider) {
- ValidationUtils.validateApplicationScope(applicationScope);
+ ValidationUtils.validateApplicationScope(appScope);
try {
- this.applicationScope = applicationScope;
-
+ this.applicationScope = appScope;
this.client = provider.getClient();
-
- this.indexName = createIndexName(config.getIndexPrefix(), applicationScope);
+ this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
+ this.indexName = IndexingUtils.createIndexName(config.getIndexPrefix(), appScope);
- this.config = config;
} catch (Exception e) {
log.error("Error setting up index", e);
throw e;
@@ -112,18 +108,34 @@ public class EsEntityIndexImpl implements EntityIndex {
AdminClient admin = client.admin();
try {
- CreateIndexResponse r = admin.indices().prepareCreate(indexName).execute().actionGet();
- log.debug("Created new Index Name [{}] ACK=[{}]", indexName, r.isAcknowledged());
+ CreateIndexResponse cir = admin.indices().prepareCreate(indexName).execute().actionGet();
+ log.debug("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
client.admin().indices().prepareRefresh(indexName).execute().actionGet();
+ XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
+ XContentFactory.jsonBuilder(), "_default_");
+
+ PutIndexTemplateResponse pitr = client.admin().indices()
+ .preparePutTemplate("usergrid_template")
+ .setTemplate(config.getIndexPrefix() + "*")
+ .addMapping("_default_", xcb)
+ .execute()
+ .actionGet();
+
+ log.debug("Create Mapping for new Index Name [{}] ACK=[{}]",
+ indexName, pitr.isAcknowledged());
+
try {
// TODO: figure out what refresh above is not enough to ensure index is ready
Thread.sleep(500);
- } catch (InterruptedException ex) {
- }
- } catch (IndexAlreadyExistsException ignored) {
- // expected
+ } catch (InterruptedException ex) {}
+
+ } catch (IndexAlreadyExistsException expected) {
+ // this is expected to happen if index already exists
+
+ } catch ( IOException ioe ) {
+ throw new RuntimeException("Error setting up index", ioe);
}
}
@@ -135,7 +147,7 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public CandidateResults search(final IndexScope indexScope, final Query query) {
- final String indexType = createCollectionScopeTypeName(indexScope);
+ final String indexType = IndexingUtils.createCollectionScopeTypeName(indexScope);
QueryBuilder qb = query.createQueryBuilder();
@@ -203,7 +215,8 @@ public class EsEntityIndexImpl implements EntityIndex {
}
log.debug("Executing query with cursor: {} ", scrollId);
- SearchScrollRequestBuilder ssrb = client.prepareSearchScroll(scrollId).setScroll(cursorTimeout + "m");
+ SearchScrollRequestBuilder ssrb =
+ client.prepareSearchScroll(scrollId).setScroll(cursorTimeout + "m");
searchResponse = ssrb.execute().actionGet();
}
@@ -234,70 +247,6 @@ public class EsEntityIndexImpl implements EntityIndex {
return candidateResults;
}
- /**
- * Build mappings for data to be indexed. Setup String fields as not_analyzed and analyzed,
- * where the analyzed field is named {name}_ug_analyzed
- *
- * @param builder Add JSON object to this builder.
- * @param type ElasticSearch type of entity.
- *
- * @return Content builder with JSON for mapping.
- *
- * @throws java.io.IOException On JSON generation error.
- */
- public static XContentBuilder createDoubleStringIndexMapping( XContentBuilder builder, String type )
- throws IOException {
-
- builder = builder
-
- .startObject()
-
- .startObject( type )
-
- .startArray( "dynamic_templates" )
-
- // any string with field name that starts with sa_ gets analyzed
- .startObject()
- .startObject( "template_1" )
- .field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "string" )
- .field( "index", "analyzed" )
- .endObject()
- .endObject()
- .endObject()
-
- // all other strings are not analyzed
- .startObject()
- .startObject( "template_2" )
- .field( "match", "*" )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" )
- .field( "type", "string" )
- .field( "index", "not_analyzed" )
- .endObject()
- .endObject()
- .endObject()
-
- // fields names starting with go_ get geo-indexed
- .startObject()
- .startObject( "template_3" )
- .field( "match", GEO_PREFIX + "location" )
- .startObject( "mapping" )
- .field( "type", "geo_point" )
- .endObject()
- .endObject()
- .endObject()
-
- .endArray()
-
- .endObject()
-
- .endObject();
-
- return builder;
- }
-
public void refresh() {
client.admin().indices().prepareRefresh(indexName).execute().actionGet();
log.debug("Refreshed index: " + indexName);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0edbe19/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 8dbaa0d..f3e1ba2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -18,12 +18,14 @@ package org.apache.usergrid.persistence.index.impl;/*
*/
+import java.io.IOException;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.elasticsearch.common.xcontent.XContentBuilder;
public class IndexingUtils {
@@ -104,4 +106,68 @@ public class IndexingUtils {
}
+ /**
+ * Build mappings for data to be indexed. Setup String fields as not_analyzed and analyzed,
+ * where the analyzed field is named {name}_ug_analyzed
+ *
+ * @param builder Add JSON object to this builder.
+ * @param type ElasticSearch type of entity.
+ *
+ * @return Content builder with JSON for mapping.
+ *
+ * @throws java.io.IOException On JSON generation error.
+ */
+ public static XContentBuilder createDoubleStringIndexMapping(
+ XContentBuilder builder, String type ) throws IOException {
+
+ builder = builder
+
+ .startObject()
+
+ .startObject( type )
+
+ .startArray( "dynamic_templates" )
+
+ // any string with field name that starts with sa_ gets analyzed
+ .startObject()
+ .startObject( "template_1" )
+ .field( "match", ANALYZED_STRING_PREFIX + "*" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "string" )
+ .field( "index", "analyzed" )
+ .endObject()
+ .endObject()
+ .endObject()
+
+ // all other strings are not analyzed
+ .startObject()
+ .startObject( "template_2" )
+ .field( "match", "*" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "not_analyzed" )
+ .endObject()
+ .endObject()
+ .endObject()
+
+ // fields names starting with go_ get geo-indexed
+ .startObject()
+ .startObject( "template_3" )
+ .field( "match", GEO_PREFIX + "location" )
+ .startObject( "mapping" )
+ .field( "type", "geo_point" )
+ .endObject()
+ .endObject()
+ .endObject()
+
+ .endArray()
+
+ .endObject()
+
+ .endObject();
+
+ return builder;
+ }
+
}