You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/03/31 01:15:06 UTC
usergrid git commit: Add concurrency to re-index and fix
de-serialization problems with Shiro cache, plus comments.
Repository: usergrid
Updated Branches:
refs/heads/release-2.1.1 8014b4f91 -> f5daca59a
Add concurrency to re-index and fix de-serialization problems with Shiro cache, plus comments.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f5daca59
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f5daca59
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f5daca59
Branch: refs/heads/release-2.1.1
Commit: f5daca59a94175a83cb03a7b413eacaa90e78589
Parents: 8014b4f
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Mar 30 16:14:59 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Mar 30 16:14:59 2016 -0700
----------------------------------------------------------------------
.../corepersistence/index/IndexProcessorFig.java | 11 ++++++++++-
.../corepersistence/index/ReIndexServiceImpl.java | 10 ++++++----
.../security/shiro/credentials/AdminUserAccessToken.java | 3 +++
.../shiro/credentials/ApplicationClientCredentials.java | 5 +++++
.../security/shiro/principals/AdminUserPrincipal.java | 3 +++
.../shiro/principals/ApplicationGuestPrincipal.java | 4 +++-
.../security/shiro/principals/ApplicationPrincipal.java | 4 +++-
.../shiro/principals/ApplicationUserPrincipal.java | 3 +++
.../security/shiro/principals/OrganizationPrincipal.java | 4 +++-
9 files changed, 39 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 15f9f9d..b94da65 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -44,6 +44,8 @@ public interface IndexProcessorFig extends GuicyFig {
String REINDEX_BUFFER_SIZE = "elasticsearch.reindex.buffer_size";
+ String REINDEX_CONCURRENCY_FACTOR = "elasticsearch.reindex.concurrency.factor";
+
/**
* Set the amount of time to wait when Elasticsearch rejects a requests before
@@ -88,11 +90,18 @@ public interface IndexProcessorFig extends GuicyFig {
@Key(ELASTICSEARCH_QUEUE_IMPL)
String getQueueImplementation();
- @Default("1000")
+ @Default("100")
@Key(REINDEX_BUFFER_SIZE)
int getReindexBufferSize();
/**
+ * The number of parallel buffers during re-index that can be processed
+ */
+ @Default("10")
+ @Key(REINDEX_CONCURRENCY_FACTOR)
+ int getReindexConcurrencyFactor();
+
+ /**
* Flag to resolve the LOCAL queue implementation service synchronously.
*/
@Default("false")
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index da2ef1d..b5f7d77 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -125,12 +125,14 @@ public class ReIndexServiceImpl implements ReIndexService {
final Observable<List<EdgeScope>> runningReIndex = allEntityIdsObservable.getEdgesToEntities( applicationScopes,
reIndexRequestBuilder.getCollectionName(), cursorSeek.getSeekValue() )
.buffer( indexProcessorFig.getReindexBufferSize())
- .doOnNext(edges -> {
+ .flatMap( edgeScopes -> Observable.just(edgeScopes)
+ .doOnNext(edges -> {
- logger.info("Sending batch of {} to be indexed.", edges.size());
- indexService.indexBatch(edges, modifiedSince);
+ logger.info("Sending batch of {} to be indexed.", edges.size());
+ indexService.indexBatch(edges, modifiedSince);
+ })
+ .subscribeOn( Schedulers.io() ), indexProcessorFig.getReindexConcurrencyFactor());
- });
//start our sampler and state persistence
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
index a2f3743..4c3ffce 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java
@@ -19,6 +19,9 @@ package org.apache.usergrid.security.shiro.credentials;
public class AdminUserAccessToken extends AbstractAccessTokenCredentials implements AdminUserCredentials {
+ /**
+ * Needed for Jackson. Do not remove
+ */
public AdminUserAccessToken() {}
public AdminUserAccessToken( String token ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
index 8c20bb6..0972c37 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java
@@ -19,6 +19,11 @@ package org.apache.usergrid.security.shiro.credentials;
public class ApplicationClientCredentials extends AbstractClientCredentials implements ApplicationCredentials {
+ /**
+ * Needed for Jackson, do not remove
+ */
+ public ApplicationClientCredentials(){}
+
public ApplicationClientCredentials( String key, String secret ) {
super( key, secret );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
index 0e18f35..a594d1e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java
@@ -37,6 +37,9 @@ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.getPermissio
public class AdminUserPrincipal extends UserPrincipal {
private static final Logger logger = LoggerFactory.getLogger(AdminUserPrincipal.class);
+ /**
+ * Needed for Jackson, do not remove
+ */
public AdminUserPrincipal() {
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
index 38da94d..9e9d821 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java
@@ -43,7 +43,9 @@ public class ApplicationGuestPrincipal extends PrincipalIdentifier {
ApplicationInfo application;
-
+ /**
+ * Needed for Jackson, do not remove
+ */
public ApplicationGuestPrincipal( ) {}
public ApplicationGuestPrincipal( ApplicationInfo application ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
index 5252052..cff9daa 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java
@@ -34,7 +34,9 @@ public class ApplicationPrincipal extends PrincipalIdentifier {
ApplicationInfo application;
-
+ /**
+ * Needed for Jackson, do not remove
+ */
public ApplicationPrincipal() {}
public ApplicationPrincipal( ApplicationInfo application ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
index 962fcf5..65503b4 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java
@@ -45,6 +45,9 @@ public class ApplicationUserPrincipal extends UserPrincipal {
private static final Logger logger = LoggerFactory.getLogger(AdminUserPrincipal.class);
+ /**
+ * Needed for Jackson, do not remove
+ */
public ApplicationUserPrincipal() {}
public ApplicationUserPrincipal( UUID applicationId, UserInfo user ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
index e885365..0ebbaa6 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java
@@ -40,7 +40,9 @@ public class OrganizationPrincipal extends PrincipalIdentifier {
OrganizationInfo organization;
-
+ /**
+ * Needed for Jackson, do not remove
+ */
public OrganizationPrincipal() {
}