You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/12/08 22:23:47 UTC
[05/50] [abbrv] incubator-usergrid git commit: Make ES store source
and store fields options configurable,
and defaults are to not store source and not store fields.
Make ES store source and store fields options configurable, and defaults are to not store source and not store fields.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/967807e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/967807e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/967807e9
Branch: refs/heads/no-source-in-es
Commit: 967807e9b63e1c3ca1042ca33138c5de05a95111
Parents: c3fc151
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Dec 3 15:19:04 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Dec 3 15:19:04 2014 -0500
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 5 +-
.../usergrid/persistence/index/IndexFig.java | 15 ++++--
.../index/impl/EsEntityIndexImpl.java | 2 +-
.../persistence/index/impl/IndexingUtils.java | 50 ++++++++++----------
4 files changed, 42 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/967807e9/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index c840d21..f7d2865 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -59,9 +59,12 @@ elasticsearch.cluster_name=elasticsearch
elasticsearch.index_prefix=usergrid
elasticsearch.hosts=127.0.0.1
elasticsearch.port=9300
-#We don't want to overwrite, let the defaults be used
+
+# We don't want to overwrite, let the defaults be used
#elasticsearch.number_shards=1
#elasticsearch.number_replicas=1
+#elasticsearch.store_source=false
+#elasticsearch.store_fields=false
index.query.limit.default=1000
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/967807e9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 0f405f3..def5e90 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -47,10 +47,11 @@ public interface IndexFig extends GuicyFig {
public static final String ELASTICSEARCH_FORCE_REFRESH = "elasticsearch.force_refresh";
+ public static final String ELASTICSEARCH_STORE_SOURCE = "elasticsearch.store_source";
- /**
- * the number of times we can fail before we refresh the client
- */
+ public static final String ELASTICSEARCH_STORE_FIELDS = "elasticsearch.store_fields";
+
+ /** The number of times we can fail before we refresh the client */
public static final String ELASTICSEARCH_FAIL_REFRESH = "elasticsearch.fail_refresh";
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
@@ -104,4 +105,12 @@ public interface IndexFig extends GuicyFig {
@Default( "20" )
@Key( ELASTICSEARCH_FAIL_REFRESH )
int getFailRefreshCount();
+
+ @Default( "false" )
+ @Key( ELASTICSEARCH_STORE_SOURCE )
+ boolean getStoreStource();
+
+ @Default( "false" )
+ @Key( ELASTICSEARCH_STORE_FIELDS )
+ boolean getStoreFields();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/967807e9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 22aa05c..1a9b152 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -215,7 +215,7 @@ public class EsEntityIndexImpl implements EntityIndex {
private void createMappings() throws IOException {
XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), "_default_" );
+ XContentFactory.jsonBuilder(), "_default_", config );
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
.preparePutTemplate( "usergrid_template" )
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/967807e9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 6272dcf..3981efa 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -143,7 +144,7 @@ public class IndexingUtils {
* @throws java.io.IOException On JSON generation error.
*/
public static XContentBuilder createDoubleStringIndexMapping(
- XContentBuilder builder, String type ) throws IOException {
+ XContentBuilder builder, String type, IndexFig config ) throws IOException {
builder = builder
@@ -154,7 +155,10 @@ public class IndexingUtils {
// don't store source object in ES
.field("_source")
.startObject()
- .field("_source").startObject().field("enabled", false).endObject()
+ .field("_source")
+ .startObject()
+ .field("enabled", config.getStoreStource())
+ .endObject()
.endObject()
.startArray( "dynamic_templates" )
@@ -162,6 +166,7 @@ public class IndexingUtils {
// we need most specific mappings first since it's a stop on match algorithm
.startObject() // array item
+
.startObject( "context_template" )
.field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
.field( "match_mapping_type", "string" )
@@ -169,8 +174,7 @@ public class IndexingUtils {
.field( "type", "string" )
.field( "index", "not_analyzed" )
.endObject()
- // dont't store field in ES
- .field("store", false)
+ .field("store", config.getStoreFields() )
.endObject()
.endObject()
@@ -183,8 +187,7 @@ public class IndexingUtils {
.field( "type", "string" )
.field( "index", "not_analyzed" )
.endObject()
- // dont't store field in ES
- .field("store", false)
+ .field("store", config.getStoreFields())
.endObject()
.endObject()
@@ -200,8 +203,21 @@ public class IndexingUtils {
.field( "type", "string" )
.field( "index", "analyzed" )
.endObject()
- // dont't store field in ES
- .field("store", false)
+ .field("store", config.getStoreFields())
+ .endObject()
+
+ .endObject()
+
+ // fields names starting with go_ get geo-indexed
+
+ .startObject() // array item
+
+ .startObject( "template_3" )
+ .field( "match", GEO_PREFIX + "location" )
+ .startObject( "mapping" )
+ .field( "type", "geo_point" )
+ .endObject()
+ .field("store", config.getStoreFields())
.endObject()
.endObject()
@@ -218,23 +234,7 @@ public class IndexingUtils {
.field( "type", "string" )
.field( "index", "not_analyzed" )
.endObject()
- // dont't store field in ES
- .field("store", false)
- .endObject()
-
- .endObject()
-
- // fields names starting with go_ get geo-indexed
-
- .startObject() // array item
-
- .startObject( "template_3" )
- .field( "match", GEO_PREFIX + "location" )
- .startObject( "mapping" )
- .field( "type", "geo_point" )
- .endObject()
- // dont't store field in ES
- .field("store", false)
+ .field("store", config.getStoreFields())
.endObject()
.endObject()