You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2014/11/26 01:37:54 UTC
[2/2] incubator-usergrid git commit: adding config based interface so
nothing is es specific, adding rest routes
adding config based interface so nothing is es specific, adding rest routes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cfa73b43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cfa73b43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cfa73b43
Branch: refs/heads/index-alias
Commit: cfa73b43a896639f5c28653b6f60d6b550549cc5
Parents: 0273c55
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Nov 25 17:37:37 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Nov 25 17:37:37 2014 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 5 +
.../HybridEntityManagerFactory.java | 5 +
.../persistence/EntityManagerFactory.java | 9 +
.../cassandra/EntityManagerFactoryImpl.java | 5 +
.../usergrid/persistence/index/EntityIndex.java | 5 +-
.../index/impl/EsEntityIndexImpl.java | 13 +-
.../persistence/index/impl/EntityIndexTest.java | 6 +-
.../org/apache/usergrid/rest/IndexResource.java | 215 +++++++++++++++++++
.../apache/usergrid/rest/SystemResource.java | 151 +------------
9 files changed, 262 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 82e80a5..5a09423 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -701,6 +701,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
@Override
+ public void addIndex(final UUID applicationId,final String indexSuffix,final Map<String,Object> config){
+ getManagerCache().getEntityIndex(CpNamingUtils.getApplicationScope( applicationId )).addIndex(indexSuffix,config);
+ }
+
+ @Override
public Health getEntityStoreHealth() {
// could use any collection scope here, does not matter
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 826d19d..0f1fbed 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -198,6 +198,11 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
}
@Override
+ public void addIndex(UUID appId, String suffix, Map<String, Object> config) {
+ factory.addIndex(appId,suffix,config);
+ }
+
+ @Override
public Health getEntityStoreHealth() {
return factory.getEntityStoreHealth();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 5dffb9f..33413a0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence;
import java.util.Map;
import java.util.UUID;
import org.apache.usergrid.persistence.core.util.Health;
+import org.apache.usergrid.persistence.index.EntityIndex;
import org.springframework.context.ApplicationContext;
@@ -160,6 +161,14 @@ public interface EntityManagerFactory {
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver object);
+ /**
+ * Add a new index to the application for scale
+ * @param appId application id
+ * @param suffix unique indentifier for additional index
+ * @param config {"replicas":num,"shards":num}
+ */
+ public void addIndex(final UUID appId,final String suffix, Map<String,Object> config);
+
public Health getEntityStoreHealth();
public interface ProgressObserver {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 8d9e239..68abf2e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -472,6 +472,11 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
}
@Override
+ public void addIndex(UUID appId, String suffix, Map<String, Object> config) {
+ throw new UnsupportedOperationException("Not supported in v1");
+ }
+
+ @Override
public Health getEntityStoreHealth() {
throw new UnsupportedOperationException("Not supported yet.");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 30dc3fc..93dd3ea 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -24,6 +24,8 @@ import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.model.entity.Id;
+import java.util.Map;
+
/**
* Provides indexing of Entities within a scope.
@@ -39,8 +41,9 @@ public interface EntityIndex {
/**
* Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
* @param indexSuffix index name
+ * @param config {"replicas":num,"shards":num}
*/
- public void createIndexAddToAlias(final String indexSuffix, final int numShards, final int numReplicas);
+ public void addIndex(final String indexSuffix, final Map<String,Object> config);
/**
* Create the index batch.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index e7cbd2e..623788e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -133,11 +134,19 @@ public class EsEntityIndexImpl implements EntityIndex {
public void initializeIndex() {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
- createIndexAddToAlias(null,numberOfShards,numberOfReplicas);
+ addIndex(null, numberOfShards, numberOfReplicas);
}
@Override
- public void createIndexAddToAlias(final String indexSuffix, final int numberOfShards, final int numberOfReplicas) {
+ public void addIndex(final String indexSuffix, final Map<String,Object> config) {
+ if(!config.containsKey("replicas") || !config.containsKey("shards")){
+ throw new IllegalArgumentException("config must contains 'replicas' and 'shards'");
+ }
+ addIndex(indexSuffix, (int) config.get("shards"),(int)config.get("replicas"));
+ }
+
+ private void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas) {
+
try {
if (!mappingsCreated.getAndSet(true)) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 0e0e81d..360347a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -127,7 +127,11 @@ public class EntityIndexTest extends BaseIT {
testQueries( indexScope, searchTypes, entityIndex );
- entityIndex.createIndexAddToAlias("v2",1,0);
+ HashMap<String,Object> config = new HashMap<>();
+ config.put("replicas",0);
+ config.put("shards",1);
+
+ entityIndex.addIndex("v2", config);
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",100,100);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
new file mode 100644
index 0000000..f95d439
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -0,0 +1,215 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.rest;
+
+import com.sun.jersey.api.json.JSONWithPadding;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * Classy class class.
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class IndexResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(IndexResource.class);
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild" )
+ public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "rebuild indexes" );
+
+
+ final EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+
+
+ @Override
+ public void onProgress( final EntityRef entity ) {
+ logger.info( "Indexing entity {}:{} ", entity.getType(), entity.getUuid() );
+ }
+
+
+ @Override
+ public long getWriteDelayTime() {
+ return 0;
+ }
+ };
+
+
+ final Thread rebuild = new Thread() {
+
+ @Override
+ public void run() {
+ logger.info( "Rebuilding all indexes" );
+
+ try {
+ emf.rebuildAllIndexes( po );
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to rebuild indexes", e );
+ }
+ }
+ };
+
+ rebuild.setName( "Index rebuild all usergrid" );
+ rebuild.setDaemon( true );
+ rebuild.start();
+
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexes( @Context UriInfo ui, @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+ final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
+ ApiResponse response = createApiResponse();
+ response.setAction( "rebuild indexes" );
+
+
+ final EntityManager em = emf.getEntityManager( appId );
+
+ final Set<String> collectionNames = em.getApplicationCollections();
+
+ final Thread rebuild = new Thread() {
+
+ @Override
+ public void run() {
+ for ( String collectionName : collectionNames )
+
+
+ {
+ rebuildCollection( appId, collectionName, delay );
+ }
+ }
+ };
+
+ rebuild.setName( String.format( "Index rebuild for app %s", appId ) );
+ rebuild.setDaemon( true );
+ rebuild.start();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
+ public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+ throws Exception {
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+ ApiResponse response = createApiResponse();
+ response.setAction( "rebuild indexes" );
+
+ final Thread rebuild = new Thread() {
+
+ public void run() {
+
+ rebuildCollection( appId, collectionName, delay );
+ }
+ };
+
+ rebuild.setName( String.format( "Index rebuild for app %s and collection %s", appId, collectionName ) );
+ rebuild.setDaemon( true );
+ rebuild.start();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+ public JSONWithPadding addIndex(@Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "indexSuffix" ) final String indexSuffix,
+ @PathParam("shards") final int shards,
+ @PathParam("replicas") final int replicas,
+ Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback) {
+ final UUID appId = UUIDUtils.tryExtractUUID(applicationIdStr);
+ emf.addIndex(appId, indexSuffix, config);
+ ApiResponse response = createApiResponse();
+ response.setAction("Add index to alias");
+ return new JSONWithPadding(response, callback);
+ }
+
+ private void rebuildCollection( final UUID applicationId, final String collectionName, final long delay ) {
+ EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+
+ @Override
+ public void onProgress( final EntityRef entity ) {
+ logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
+ }
+ @Override
+ public long getWriteDelayTime() {
+ return delay;
+ }
+ };
+
+
+ logger.info( "Reindexing for app id: {} and collection {}", applicationId, collectionName );
+
+ emf.rebuildCollectionIndex( applicationId, collectionName, po );
+ emf.refreshIndex();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cfa73b43/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
index 1961aa1..fca3333 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
@@ -117,155 +117,10 @@ public class SystemResource extends AbstractContextResource {
}
@Path( "migrate" )
- public MigrateResource doSomething(){
+ public MigrateResource migrate(){
return getSubResource( MigrateResource.class );
}
- @RequireSystemAccess
- @PUT
- @Path( "index/rebuild" )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes" );
-
-
- final ProgressObserver po = new ProgressObserver() {
-
-
- @Override
- public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{} ", entity.getType(), entity.getUuid() );
- }
-
-
- @Override
- public long getWriteDelayTime() {
- return 0;
- }
- };
-
-
- final Thread rebuild = new Thread() {
-
- @Override
- public void run() {
- logger.info( "Rebuilding all indexes" );
-
- try {
- emf.rebuildAllIndexes( po );
- }
- catch ( Exception e ) {
- logger.error( "Unable to rebuild indexes", e );
- }
- }
- };
-
- rebuild.setName( "Index rebuild all usergrid" );
- rebuild.setDaemon( true );
- rebuild.start();
-
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui, @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
-
- throws Exception {
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes" );
-
-
- final EntityManager em = emf.getEntityManager( appId );
-
- final Set<String> collectionNames = em.getApplicationCollections();
-
- final Thread rebuild = new Thread() {
-
- @Override
- public void run() {
- for ( String collectionName : collectionNames )
-
-
- {
- rebuildCollection( appId, collectionName, delay );
- }
- }
- };
-
- rebuild.setName( String.format( "Index rebuild for app %s", appId ) );
- rebuild.setDaemon( true );
- rebuild.start();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "index/rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexes( @Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
- throws Exception {
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
- ApiResponse response = createApiResponse();
- response.setAction( "rebuild indexes" );
-
- final Thread rebuild = new Thread() {
-
- public void run() {
-
- rebuildCollection( appId, collectionName, delay );
- }
- };
-
- rebuild.setName( String.format( "Index rebuild for app %s and collection %s", appId, collectionName ) );
- rebuild.setDaemon( true );
- rebuild.start();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- private void rebuildCollection( final UUID applicationId, final String collectionName, final long delay ) {
- EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
-
- @Override
- public void onProgress( final EntityRef entity ) {
- logger.info( "Indexing entity {}:{}", entity.getType(), entity.getUuid() );
- }
-
-
- @Override
- public long getWriteDelayTime() {
- return delay;
- }
- };
-
-
- logger.info( "Reindexing for app id: {} and collection {}", applicationId, collectionName );
-
- emf.rebuildCollectionIndex( applicationId, collectionName, po );
- emf.refreshIndex();
- }
+ @Path( "index" )
+ public IndexResource index() { return getSubResource(IndexResource.class); }
}