You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/06/12 01:07:00 UTC
[08/11] incubator-usergrid git commit: add index buckets
add index buckets
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1cd23fcc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1cd23fcc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1cd23fcc
Branch: refs/heads/two-dot-o-dev
Commit: 1cd23fcc96e30b64c10f1ab88f8ba0e10cb18dca
Parents: fb33e94
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Jun 11 14:41:01 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Jun 11 14:41:01 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 1 -
.../index/ApplicationIndexLocationStrategy.java | 32 ++++++++++++--------
.../index/IndexLocationStrategyFactoryImpl.java | 6 ++--
.../index/ManagementIndexLocationStrategy.java | 2 +-
.../corepersistence/index/IndexNamingTest.java | 16 ++++++----
.../usergrid/persistence/index/EntityIndex.java | 11 ++++---
.../index/IndexLocationStrategy.java | 11 +++++--
.../index/impl/EsEntityIndexImpl.java | 26 ++++++++--------
.../index/impl/TestIndexIdentifier.java | 2 +-
9 files changed, 62 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 36d5ed3..36d55a3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2850,7 +2850,6 @@ public class CpEntityManager implements EntityManager {
return applicationId;
}
-
@Override
public IndexBucketLocator getIndexBucketLocator() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java
index 0c8a720..ad0f54d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ApplicationIndexLocationStrategy.java
@@ -19,12 +19,8 @@
*/
package org.apache.usergrid.corepersistence.index;
-import com.google.common.hash.Funnel;
-import com.google.common.hash.PrimitiveSink;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.shard.ExpandingShardLocator;
-import org.apache.usergrid.persistence.core.shard.StringHashUtils;
import org.apache.usergrid.persistence.index.IndexAlias;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexLocationStrategy;
@@ -38,8 +34,9 @@ class ApplicationIndexLocationStrategy implements IndexLocationStrategy {
private final IndexFig indexFig;
private final ApplicationScope applicationScope;
private final ApplicationIndexBucketLocator applicationIndexBucketLocator;
- private final String indexName;
+ private final String indexBucketName;
private final IndexAlias alias;
+ private final String indexRootName;
public ApplicationIndexLocationStrategy(final CassandraFig cassandraFig,
final IndexFig indexFig,
@@ -50,9 +47,10 @@ class ApplicationIndexLocationStrategy implements IndexLocationStrategy {
this.indexFig = indexFig;
this.applicationScope = applicationScope;
this.applicationIndexBucketLocator = applicationIndexBucketLocator;
- String prefix = getPrefix(); //TODO: add hash buckets by app scope
- this.alias = new ApplicationIndexAlias(indexFig, applicationScope, prefix);
- this.indexName = prefix + "_" + applicationIndexBucketLocator.getBucket(applicationScope);
+ this.indexRootName = getPrefix(); //TODO: add hash buckets by app scope
+ this.alias = new ApplicationIndexAlias(indexFig, applicationScope, indexRootName);
+
+ this.indexBucketName = indexRootName + "_" + applicationIndexBucketLocator.getBucket(applicationScope);
}
@@ -72,14 +70,22 @@ class ApplicationIndexLocationStrategy implements IndexLocationStrategy {
* @return
*/
@Override
- public String getInitialIndexName() {
- return indexName;
+ public String getIndexRootName() {
+ return indexBucketName;
}
+ /**
+ * Get index name, send in additional parameter to add incremental indexes
+ * @return
+ */
+ @Override
+ public String getIndexBucketName() {
+ return indexBucketName;
+ }
@Override
public String toString() {
- return "index id: "+indexName;
+ return "index id: "+ indexBucketName;
}
@Override
@@ -105,14 +111,14 @@ class ApplicationIndexLocationStrategy implements IndexLocationStrategy {
ApplicationIndexLocationStrategy that = (ApplicationIndexLocationStrategy) o;
if (!applicationScope.equals(that.applicationScope)) return false;
- return indexName.equals(that.indexName);
+ return indexBucketName.equals(that.indexBucketName);
}
@Override
public int hashCode() {
int result = applicationScope.hashCode();
- result = 31 * result + indexName.hashCode();
+ result = 31 * result + indexBucketName.hashCode();
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java
index 6ddf9a8..6fc0a32 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexLocationStrategyFactoryImpl.java
@@ -32,20 +32,22 @@ import org.apache.usergrid.persistence.index.IndexLocationStrategy;
public class IndexLocationStrategyFactoryImpl implements IndexLocationStrategyFactory {
private final CassandraFig cassandraFig;
private final IndexFig indexFig;
+ private final ApplicationIndexBucketLocator applicationLocatorBucketStrategy;
private final CoreIndexFig coreIndexFig;
@Inject
- public IndexLocationStrategyFactoryImpl(final CassandraFig cassandraFig, final IndexFig indexFig, final CoreIndexFig coreIndexFig){
+ public IndexLocationStrategyFactoryImpl(final CassandraFig cassandraFig, final IndexFig indexFig, final ApplicationIndexBucketLocator applicationLocatorBucketStrategy, final CoreIndexFig coreIndexFig){
this.cassandraFig = cassandraFig;
this.indexFig = indexFig;
+ this.applicationLocatorBucketStrategy = applicationLocatorBucketStrategy;
this.coreIndexFig = coreIndexFig;
}
public IndexLocationStrategy getIndexLocationStrategy(ApplicationScope applicationScope){
if(CpNamingUtils.getManagementApplicationId().equals(applicationScope.getApplication())){
return new ManagementIndexLocationStrategy(indexFig,coreIndexFig);
}
- return new ApplicationIndexLocationStrategy(cassandraFig,indexFig,applicationScope);
+ return new ApplicationIndexLocationStrategy(cassandraFig,indexFig,applicationScope, applicationLocatorBucketStrategy);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java
index 5437bc0..cfbf409 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ManagementIndexLocationStrategy.java
@@ -52,7 +52,7 @@ class ManagementIndexLocationStrategy implements IndexLocationStrategy {
}
@Override
- public String getInitialIndexName() {
+ public String getIndexRootName() {
return indexName;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java
index 094a0c2..4822005 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexNamingTest.java
@@ -23,6 +23,7 @@ import com.google.inject.Inject;
import net.jcip.annotations.NotThreadSafe;
import org.apache.usergrid.corepersistence.TestIndexModule;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.test.UseModules;
@@ -58,6 +59,9 @@ public class IndexNamingTest {
@Inject
public IndexLocationStrategyFactory indexLocationStrategyFactory;
+ @Inject
+ public ApplicationIndexBucketLocator bucketLocator;
+
private ApplicationScope applicationScope;
private ApplicationScope managementApplicationScope;
private ApplicationIndexLocationStrategy applicationLocationStrategy;
@@ -68,23 +72,23 @@ public class IndexNamingTest {
this.applicationScope = CpNamingUtils.getApplicationScope(UUID.randomUUID());
this.managementApplicationScope = CpNamingUtils.getApplicationScope(CpNamingUtils.getManagementApplicationId().getUuid());
this.managementLocationStrategy = new ManagementIndexLocationStrategy(indexFig, indexProcessorFig);
- this.applicationLocationStrategy = new ApplicationIndexLocationStrategy(cassandraFig,indexFig,applicationScope);
+ this.applicationLocationStrategy = new ApplicationIndexLocationStrategy(cassandraFig,indexFig,applicationScope,bucketLocator);
}
@Test
public void managementNaming(){
IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(managementApplicationScope);
- assertEquals(indexLocationStrategy.getInitialIndexName(),managementLocationStrategy.getInitialIndexName());
- assertEquals(indexLocationStrategy.getInitialIndexName(),indexProcessorFig.getManagementAppIndexName());
+ assertEquals(indexLocationStrategy.getIndexRootName(),managementLocationStrategy.getIndexRootName());
+ assertEquals(indexLocationStrategy.getIndexRootName(),indexProcessorFig.getManagementAppIndexName());
}
@Test
public void applicationNaming(){
IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope);
- assertEquals(indexLocationStrategy.getInitialIndexName(),applicationLocationStrategy.getInitialIndexName());
+ assertEquals(indexLocationStrategy.getIndexRootName(),applicationLocationStrategy.getIndexRootName());
- assertTrue(indexLocationStrategy.getInitialIndexName().contains(indexFig.getIndexPrefix()));
- assertTrue(indexLocationStrategy.getInitialIndexName().contains(cassandraFig.getApplicationKeyspace().toLowerCase()));
+ assertTrue(indexLocationStrategy.getIndexRootName().contains(indexFig.getIndexPrefix()));
+ assertTrue(indexLocationStrategy.getIndexRootName().contains(cassandraFig.getApplicationKeyspace().toLowerCase()));
assertTrue(indexLocationStrategy.getAlias().getReadAlias().contains(applicationScope.getApplication().getUuid().toString().toLowerCase()));
assertTrue(indexLocationStrategy.getAlias().getWriteAlias().contains(applicationScope.getApplication().getUuid().toString().toLowerCase()));
assertTrue(indexLocationStrategy.getAlias().getWriteAlias().contains("write"));
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 282d162..f19d38e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -44,7 +44,12 @@ public interface EntityIndex extends CPManager {
* @param replicas
* @param writeConsistency
*/
- void addIndex(final Optional<String> indexSuffix, final int shards, final int replicas, final String writeConsistency);
+ void addIndex(
+ final Optional<String> indexSuffix,
+ final int shards,
+ final int replicas,
+ final String writeConsistency
+ );
/**
* Refresh the index.
@@ -123,10 +128,6 @@ public interface EntityIndex extends CPManager {
*/
String[] getIndexes();
- /**
- * Add alias to index, will remove old index from write alias
- */
- void addAlias( );
/**
* type of alias
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java
index 0f2dc0c..e00101b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexLocationStrategy.java
@@ -31,10 +31,17 @@ public interface IndexLocationStrategy {
IndexAlias getAlias();
/**
- * get index name from suffix
+ * get index name
* @return
*/
- String getInitialIndexName();
+ String getIndexRootName();
+
+
+
+
+ default String getIndexBucketName() {
+ return getIndexRootName();
+ }
/**
* return unique string
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index a31c62b..7c412cf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -164,26 +164,28 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
private boolean shouldInitialize() {
- String[] reads = getIndexes(EntityIndex.AliasType.Read);
- String[] writes = getIndexes(EntityIndex.AliasType.Write);
- return reads.length==0 || writes.length==0;
+ String[] reads = getIndexes();
+ return reads.length==0;
}
@Override
- public void addIndex(final com.google.common.base.Optional<String> indexNameOverride, final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
+ public void addIndex(final Optional<String> indexNameOverride,
+ final int numberOfShards,
+ final int numberOfReplicas,
+ final String writeConsistency
+ ) {
try {
//get index name with suffix attached
- final String indexName = indexNameOverride.or(indexLocationStrategy.getInitialIndexName()) ;
+ final String indexName = indexNameOverride.or( indexLocationStrategy.getIndexBucketName() ) ;
//Create index
try {
final AdminClient admin = esProvider.getClient().admin();
Settings settings = ImmutableSettings.settingsBuilder()
- .put("index.number_of_shards", numberOfShards)
+ .put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", numberOfReplicas)
-
//dont' allow unmapped queries, and don't allow dynamic mapping
- .put("index.query.parse.allow_unmapped_fields", false)
+ .put("index.query.parse.allow_unmapped_fields", false)
.put("index.mapper.dynamic", false)
.put("action.write_consistency", writeConsistency)
.build();
@@ -215,8 +217,6 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
addAlias();
-
-
testNewIndex();
} catch (IndexAlreadyExistsException expected) {
@@ -226,13 +226,11 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
}
-
- @Override
- public void addAlias() {
+ private void addAlias() {
Timer.Context timer = updateAliasTimer.time();
try {
Boolean isAck;
- String indexName = indexLocationStrategy.getInitialIndexName();
+ String indexName = indexLocationStrategy.getIndexRootName();
final AdminClient adminClient = esProvider.getClient().admin();
String[] indexNames = getIndexes(AliasType.Write);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1cd23fcc/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java
index 5d4d3ae..82c7b98 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/TestIndexIdentifier.java
@@ -71,7 +71,7 @@ public class TestIndexIdentifier implements IndexLocationStrategy {
* @return
*/
@Override
- public String getInitialIndexName() {
+ public String getIndexRootName() {
return prefix;