You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/06/10 01:18:11 UTC
incubator-usergrid git commit: adding extra config for management
indexes
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-652 80cf5fcac -> eff7474cc
adding extra config for management indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/eff7474c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/eff7474c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/eff7474c
Branch: refs/heads/USERGRID-652
Commit: eff7474cc448c8cb2b51de45cfb3bf6706591da7
Parents: 80cf5fc
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Jun 9 17:16:00 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Jun 9 17:16:00 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 3 ++
.../corepersistence/CpManagerCache.java | 3 +-
.../index/ApplicationIndexLocationStrategy.java | 16 ++++++-
.../corepersistence/index/CoreIndexFig.java | 46 ++++++++++++++++++++
.../index/IndexLocationStrategyFactory.java | 5 +++
.../index/IndexLocationStrategyFactoryImpl.java | 6 ++-
.../index/ManagementIndexLocationStrategy.java | 16 ++++++-
.../corepersistence/index/IndexNamingTest.java | 20 +++++++--
.../usergrid/persistence/index/IndexFig.java | 6 +--
.../index/IndexLocationStrategy.java | 18 +++++++-
.../index/impl/EsEntityIndexImpl.java | 9 ++--
.../index/impl/TestIndexIdentifier.java | 10 +++++
12 files changed, 141 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 273e6bb..8d98ae2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -146,6 +146,9 @@ public class CoreModule extends AbstractModule {
install(new GuicyFigModule(IndexProcessorFig.class));
+ install(new GuicyFigModule(CoreIndexFig.class));
+
+
install( new GuicyFigModule( ApplicationIdCacheFig.class ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
index bb02449..73062b0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
@@ -71,7 +71,8 @@ public class CpManagerCache implements ManagerCache {
@Override
public AliasedEntityIndex getEntityIndex( ApplicationScope applicationScope) {
- IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope);
+ IndexLocationStrategy indexLocationStrategy
+ = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope);
return eif.createEntityIndex( indexLocationStrategy );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java
index b524e90..d4867c0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java
@@ -34,6 +34,7 @@ class ApplicationIndexLocationStrategy implements IndexLocationStrategy {
private final IndexFig indexFig;
private final ApplicationScope applicationScope;
private final String prefix;
+ private final String aliasPrefix;
public ApplicationIndexLocationStrategy(final CassandraFig cassandraFig,
final IndexFig indexFig,
@@ -43,9 +44,11 @@ class ApplicationIndexLocationStrategy implements IndexLocationStrategy {
this.indexFig = indexFig;
this.applicationScope = applicationScope;
this.prefix = getPrefix();
+ this.aliasPrefix = prefix + "_" + applicationScope.getApplication().getUuid();
}
private String getPrefix() {
+ //TODO: add hash buckets by app scope
//remove usergrid
final String indexPrefixConfig = StringUtils.isNotEmpty(indexFig.getIndexPrefix())
? indexFig.getIndexPrefix().toLowerCase() ////use lowercase value
@@ -64,7 +67,8 @@ class ApplicationIndexLocationStrategy implements IndexLocationStrategy {
*/
@Override
public IndexAlias getAlias() {
- return new IndexAlias(indexFig,prefix);
+ //TODO: include appid
+ return new IndexAlias(indexFig,aliasPrefix);
}
/**
@@ -93,6 +97,16 @@ class ApplicationIndexLocationStrategy implements IndexLocationStrategy {
}
@Override
+ public int getNumberOfShards() {
+ return indexFig.getNumberOfShards();
+ }
+
+ @Override
+ public int getNumberOfReplicas() {
+ return indexFig.getNumberOfReplicas();
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CoreIndexFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CoreIndexFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CoreIndexFig.java
new file mode 100644
index 0000000..f6dadfb
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CoreIndexFig.java
@@ -0,0 +1,46 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.corepersistence.index;
+
+import org.safehaus.guicyfig.Default;
+import org.safehaus.guicyfig.GuicyFig;
+import org.safehaus.guicyfig.Key;
+
+/**
+ * configuration for index in core
+ */
+public interface CoreIndexFig extends GuicyFig {
+
+ String ELASTICSEARCH_MANAGEMENT_NUMBER_OF_SHARDS = "elasticsearch.management_number_shards";
+
+ String ELASTICSEARCH_MANAGEMENT_NUMBER_OF_REPLICAS = "elasticsearch.management_number_replicas";
+
+ @Default( "6" )
+ @Key( ELASTICSEARCH_MANAGEMENT_NUMBER_OF_SHARDS )
+ int getManagementNumberOfShards();
+
+ @Default( "1" )
+ @Key( ELASTICSEARCH_MANAGEMENT_NUMBER_OF_REPLICAS )
+ int getManagementNumberOfReplicas();
+
+ @Default( "usergrid_management" )
+ @Key( "elasticsearch.managment_index" )
+ String getManagementAppIndexName();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactory.java
index d30dc6b..91f4938 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactory.java
@@ -26,5 +26,10 @@ import org.apache.usergrid.persistence.index.IndexLocationStrategy;
* Retrieve index locations by app scope
*/
public interface IndexLocationStrategyFactory {
+ /**
+ * given app scope find the location strategy
+ * @param applicationScope
+ * @return
+ */
IndexLocationStrategy getIndexLocationStrategy(ApplicationScope applicationScope);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java
index 40325c3..6ddf9a8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java
@@ -32,16 +32,18 @@ import org.apache.usergrid.persistence.index.IndexLocationStrategy;
public class IndexLocationStrategyFactoryImpl implements IndexLocationStrategyFactory {
private final CassandraFig cassandraFig;
private final IndexFig indexFig;
+ private final CoreIndexFig coreIndexFig;
@Inject
- public IndexLocationStrategyFactoryImpl(final CassandraFig cassandraFig, final IndexFig indexFig){
+ public IndexLocationStrategyFactoryImpl(final CassandraFig cassandraFig, final IndexFig indexFig, final CoreIndexFig coreIndexFig){
this.cassandraFig = cassandraFig;
this.indexFig = indexFig;
+ this.coreIndexFig = coreIndexFig;
}
public IndexLocationStrategy getIndexLocationStrategy(ApplicationScope applicationScope){
if(CpNamingUtils.getManagementApplicationId().equals(applicationScope.getApplication())){
- return new ManagementIndexLocationStrategy(indexFig);
+ return new ManagementIndexLocationStrategy(indexFig,coreIndexFig);
}
return new ApplicationIndexLocationStrategy(cassandraFig,indexFig,applicationScope);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java
index 76d28fc..6be05c9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java
@@ -35,12 +35,14 @@ class ManagementIndexLocationStrategy implements IndexLocationStrategy {
private final Id managementAppId;
private final String prefix;
private final IndexFig indexFig;
+ private final CoreIndexFig coreIndexFig;
- public ManagementIndexLocationStrategy(IndexFig indexFig){
+ public ManagementIndexLocationStrategy(final IndexFig indexFig, final CoreIndexFig coreIndexFig){
this.indexFig = indexFig;
+ this.coreIndexFig = coreIndexFig;
this.managementAppId = CpNamingUtils.getManagementApplicationId();
//remove usergrid
- this.prefix = indexFig.getManagementAppIndexName().toLowerCase(); ////use lowercase value
+ this.prefix = coreIndexFig.getManagementAppIndexName().toLowerCase(); ////use lowercase value
}
@@ -64,6 +66,16 @@ class ManagementIndexLocationStrategy implements IndexLocationStrategy {
}
@Override
+ public int getNumberOfShards() {
+ return coreIndexFig.getManagementNumberOfShards();
+ }
+
+ @Override
+ public int getNumberOfReplicas() {
+ return coreIndexFig.getManagementNumberOfReplicas();
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java
index 893359e..82fb154 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java
@@ -37,6 +37,7 @@ import org.junit.runner.RunWith;
import java.util.UUID;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
@@ -52,6 +53,10 @@ public class IndexNamingTest {
@Inject
public IndexFig indexFig;
+
+ @Inject
+ public CoreIndexFig indexProcessorFig;
+
@Inject
public IndexLocationStrategyFactory indexLocationStrategyFactory;
@@ -64,20 +69,29 @@ public class IndexNamingTest {
public void setup(){
this.applicationScope = CpNamingUtils.getApplicationScope(UUID.randomUUID());
this.managementApplicationScope = CpNamingUtils.getApplicationScope(CpNamingUtils.getManagementApplicationId().getUuid());
- this.managementLocationStrategy = new ManagementIndexLocationStrategy(indexFig);
+ this.managementLocationStrategy = new ManagementIndexLocationStrategy(indexFig, indexProcessorFig);
this.applicationLocationStrategy = new ApplicationIndexLocationStrategy(cassandraFig,indexFig,applicationScope);
}
@Test
public void managementNaming(){
IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(managementApplicationScope);
- assertTrue(indexLocationStrategy.getIndex(null).equals(managementLocationStrategy.getIndex(null)));
+ assertEquals(indexLocationStrategy.getIndex(null),managementLocationStrategy.getIndex(null));
+ assertEquals(indexLocationStrategy.getIndex(null),indexProcessorFig.getManagementAppIndexName());
}
@Test
public void applicationNaming(){
IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope);
- assertTrue(indexLocationStrategy.getIndex(null).equals(applicationLocationStrategy.getIndex(null)));
+ assertEquals(indexLocationStrategy.getIndex(null),applicationLocationStrategy.getIndex(null));
+
+ assertTrue(indexLocationStrategy.getIndex(null).contains(indexFig.getIndexPrefix()));
+ assertTrue(indexLocationStrategy.getIndex(null).contains(cassandraFig.getApplicationKeyspace().toLowerCase()));
+ assertTrue(indexLocationStrategy.getAlias().getReadAlias().contains(applicationScope.getApplication().getUuid().toString().toLowerCase()));
+ assertTrue(indexLocationStrategy.getAlias().getWriteAlias().contains(applicationScope.getApplication().getUuid().toString().toLowerCase()));
+ assertTrue(indexLocationStrategy.getAlias().getWriteAlias().contains("write"));
+ assertTrue(indexLocationStrategy.getAlias().getReadAlias().contains("read"));
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 5c61928..40662c4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -140,6 +140,8 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_NUMBER_OF_REPLICAS )
int getNumberOfReplicas();
+
+
@Default( "20" )
@Key( ELASTICSEARCH_FAIL_REFRESH )
int getFailRefreshCount();
@@ -199,7 +201,5 @@ public interface IndexFig extends GuicyFig {
@Key( ELASTICSEARCH_WRITE_TIMEOUT )
long getWriteTimeout();
- @Default( "usergrid_management" )
- @Key( "elasticsearch.managment_index" )
- String getManagementAppIndexName();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java
index b779ae9..633a109 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.persistence.index;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
/**
- * Classy class class.
+ * location strategy for index
*/
public interface IndexLocationStrategy {
/**
@@ -43,5 +43,21 @@ public interface IndexLocationStrategy {
*/
String toString();
+ /**
+ * only used by search types
+ * @return
+ */
ApplicationScope getApplicationScope();
+
+ /**
+ * number of shards for default
+ * @return
+ */
+ int getNumberOfShards();
+
+ /**
+ * default replicas
+ * @return
+ */
+ int getNumberOfReplicas();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 590baf9..5cb002e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -132,7 +132,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
final IndexRefreshCommand indexRefreshCommand,
final MetricsFactory metricsFactory,
final IndexBufferConsumer indexBatchBufferProducer,
- @Assisted final IndexLocationStrategy indexLocationStrategy
+ final IndexLocationStrategy indexLocationStrategy
) {
this.indexFig = indexFig;
@@ -146,7 +146,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
this.applicationScope = indexLocationStrategy.getApplicationScope();
this.cursorTimeout = indexFig.getQueryCursorTimeout();
this.queryTimeout = indexFig.getWriteTimeout();
- this.searchRequest = new SearchRequestBuilderStrategy( esProvider, applicationScope, alias, cursorTimeout );
+ this.searchRequest
+ = new SearchRequestBuilderStrategy( esProvider, applicationScope, alias, cursorTimeout );
this.searchRequestBuilderStrategyV2 = new SearchRequestBuilderStrategyV2( esProvider, applicationScope, alias, cursorTimeout );
this.addTimer = metricsFactory
@@ -166,8 +167,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
@Override
public void initialize() {
- final int numberOfShards = indexFig.getNumberOfShards();
- final int numberOfReplicas = indexFig.getNumberOfReplicas();
+ final int numberOfShards = indexLocationStrategy.getNumberOfShards();
+ final int numberOfReplicas = indexLocationStrategy.getNumberOfReplicas();
if (shouldInitialize()) {
addIndex( null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eff7474c/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java
index adbdfbe..eb2f7f8 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java
@@ -90,4 +90,14 @@ public class TestIndexIdentifier implements IndexLocationStrategy {
public ApplicationScope getApplicationScope() {
return applicationScope;
}
+
+ @Override
+ public int getNumberOfShards() {
+ return indexFig.getNumberOfShards();
+ }
+
+ @Override
+ public int getNumberOfReplicas() {
+ return indexFig.getNumberOfReplicas();
+ }
}