You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by md...@apache.org on 2018/09/05 06:38:59 UTC
usergrid git commit: Fixed failing tests and bugs -Fixed tests for
Core module that were broken due to changes in 8913e43 -Fixed failing tests
for Services module related to threading issues -Fixed failing tests for Rest
module related to Index rebuild -
Repository: usergrid
Updated Branches:
refs/heads/master 8913e43d2 -> faafed885
Fixed failing tests and bugs
-Fixed tests for Core module that were broken due to changes in 8913e43
-Fixed failing tests for Services module related to threading issues
-Fixed failing tests for Rest module related to Index rebuild
-Fixed re-index code to ensure rebuild status is correctly maintained
-Updated Shiro to 1.3.2 (latest)
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/faafed88
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/faafed88
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/faafed88
Branch: refs/heads/master
Commit: faafed8858d9999195337f1776a94d57f84ac7be
Parents: 8913e43
Author: Keyur Karnik <ke...@gmail.com>
Authored: Wed Aug 29 05:00:13 2018 -0700
Committer: Keyur Karnik <ke...@gmail.com>
Committed: Tue Sep 4 21:25:31 2018 -0700
----------------------------------------------------------------------
.../corepersistence/index/ReIndexService.java | 2 +-
.../index/ReIndexServiceImpl.java | 22 ++++----
.../usergrid/persistence/RebuildIndexTest.java | 57 ++++++++++++++++----
stack/pom.xml | 2 +-
.../apache/usergrid/rest/IndexResourceIT.java | 45 +++++++++++-----
.../org/apache/usergrid/NewOrgAppAdminRule.java | 1 +
.../management/OrganizationConfigIT.java | 4 +-
.../management/export/ExportServiceIT.java | 3 ++
8 files changed, 98 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index d37f117..48c3908 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -97,7 +97,7 @@ public interface ReIndexService {
}
/**
- * Get the jobId used to resume this operation
+ * Get the collectionName used to resume this operation
*/
public String getCollectionName() {
return collectionName;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index d4fb249..036f89c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -202,7 +202,7 @@ public class ReIndexServiceImpl implements ReIndexService {
.subscribeOn( Schedulers.io() ).subscribe();
if(isForCollection){
- return new ReIndexStatus( "", Status.STARTED, 0, 0, reIndexRequestBuilder.getCollectionName().get() );
+ return new ReIndexStatus( "", Status.STARTED, 0, 0, CpNamingUtils.getNameFromEdgeType(reIndexRequestBuilder.getCollectionName().get()) );
}
@@ -365,21 +365,23 @@ public class ReIndexServiceImpl implements ReIndexService {
private void writeStateMetaForCollection(final String appIdString, final String collectionName,
final Status status, final long processedCount, final long lastUpdated ) {
- if(logger.isDebugEnabled()) {
+ String prefixedColName = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName.toLowerCase() );
+ if(logger.isDebugEnabled()) {
logger.debug( "Flushing state for collection {}, status {}, processedCount {}, lastUpdated {}",
- collectionName, status, processedCount, lastUpdated);
+ collectionName, status, processedCount, lastUpdated);
}
- mapManager.putString( appIdString + MAP_SEPARATOR + collectionName + MAP_STATUS_KEY, status.name() );
- mapManager.putLong( appIdString + MAP_SEPARATOR + collectionName + MAP_COUNT_KEY, processedCount );
- mapManager.putLong( appIdString + MAP_SEPARATOR + collectionName + MAP_UPDATED_KEY, lastUpdated );
+ mapManager.putString( appIdString + MAP_SEPARATOR + prefixedColName + MAP_STATUS_KEY, status.name() );
+ mapManager.putLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_COUNT_KEY, processedCount );
+ mapManager.putLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_UPDATED_KEY, lastUpdated );
}
private ReIndexStatus getIndexResponseForCollection( final String appIdString, final String collectionName ) {
- final String stringStatus =
- mapManager.getString( appIdString + MAP_SEPARATOR + collectionName + MAP_STATUS_KEY );
+ String prefixedColName = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName.toLowerCase() );
+ final String stringStatus =
+ mapManager.getString( appIdString + MAP_SEPARATOR + prefixedColName + MAP_STATUS_KEY );
if(stringStatus == null){
return new ReIndexStatus( "", Status.UNKNOWN, 0, 0, collectionName );
@@ -387,8 +389,8 @@ public class ReIndexServiceImpl implements ReIndexService {
final Status status = Status.valueOf( stringStatus );
- final long processedCount = mapManager.getLong( appIdString + MAP_SEPARATOR + collectionName + MAP_COUNT_KEY );
- final long lastUpdated = mapManager.getLong( appIdString + MAP_SEPARATOR + collectionName + MAP_UPDATED_KEY );
+ final long processedCount = mapManager.getLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_COUNT_KEY );
+ final long lastUpdated = mapManager.getLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_UPDATED_KEY );
return new ReIndexStatus( "", status, processedCount, lastUpdated, collectionName );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index 57962c0..90af5ba 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -154,12 +154,12 @@ public class RebuildIndexTest extends AbstractCoreIT {
ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder);
- assertNotNull(status.getJobId(), "JobId is present");
+ assertNotNull(status.getCollectionName(), "Collection name is present");
logger.info("Rebuilt index");
- waitForRebuild(status, reIndexService);
+ waitForRebuild(em.getApplicationId().toString(), status.getCollectionName(), reIndexService);
//app.waitForQueueDrainAndRefreshIndex(15000);
@@ -279,7 +279,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
logger.info("Rebuilt index, jobID={}", status.getJobId());
- waitForRebuild(status, reIndexService);
+ waitForRebuild(status.getJobId(), reIndexService);
logger.info("Rebuilt index");
@@ -387,7 +387,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
logger.info("Rebuilt index");
- waitForRebuild(status, reIndexService);
+ waitForRebuild(status.getJobId(), reIndexService);
logger.info("Rebuilt index");
@@ -486,7 +486,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
logger.info("Rebuilt index");
- waitForRebuild(status, reIndexService);
+ waitForRebuild(status.getJobId(), reIndexService);
logger.info("Rebuilt index");
@@ -505,18 +505,53 @@ public class RebuildIndexTest extends AbstractCoreIT {
/**
* Wait for the rebuild to occur
*/
- private void waitForRebuild(final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService)
+ private void waitForRebuild(final String jobId, final ReIndexService reIndexService)
throws InterruptedException, IllegalArgumentException {
- if (status != null) {
- logger.info("waitForRebuild: jobID={}", status.getJobId());
+ if (jobId != null && !jobId.trim().equals("")) {
+ logger.info("waitForRebuild: jobID={}", jobId);
} else {
- logger.info("waitForRebuild: error, status = null");
- throw new IllegalArgumentException("reindexStatus = null");
+ logger.info("waitForRebuild: error, jobId = null or empty");
+ throw new IllegalArgumentException("jobId = null or empty");
}
while (true) {
try {
- final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(status.getJobId());
+ final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(jobId);
+
+ if (updatedStatus == null) {
+ logger.info("waitForRebuild: updated status is null");
+ } else {
+ logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed());
+
+ if (updatedStatus.getStatus() == ReIndexService.Status.COMPLETE) {
+ break;
+ }
+ }
+ } catch (IllegalArgumentException iae) {
+ //swallow. Thrown if our job can't be found. I.E hasn't updated yet
+ }
+
+
+ Thread.sleep(1000);
+ }
+ }
+
+
+ /**
+ * Wait for the rebuild to occur
+ */
+ private void waitForRebuild(final String appId, final String collectionName, final ReIndexService reIndexService)
+ throws InterruptedException, IllegalArgumentException {
+ if (appId != null && !appId.trim().equals("") && collectionName != null && !collectionName.trim().equals("")) {
+ logger.info("waitForRebuild: appId={} collName={}", appId, collectionName);
+ } else {
+ logger.info("waitForRebuild: error, appId or collName = null or empty");
+ throw new IllegalArgumentException("appId or collName = null or empty");
+ }
+ while (true) {
+
+ try {
+ final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatusForCollection(appId, collectionName);
if (updatedStatus == null) {
logger.info("waitForRebuild: updated status is null");
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index c98c72d..aa289d0 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -114,7 +114,7 @@
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
<org.springframework.version>3.2.13.RELEASE</org.springframework.version>
- <shiro-version>1.2.4</shiro-version>
+ <shiro-version>1.3.2</shiro-version>
<slf4j-version>1.7.2</slf4j-version>
<snakeyaml-version>1.9</snakeyaml-version>
<tomcat-version>7.0.64</tomcat-version>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
index 28d6501..be71881 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java
@@ -25,6 +25,7 @@ import org.apache.usergrid.rest.test.resource.AbstractRestIT;
import org.apache.usergrid.rest.test.resource.model.ApiResponse;
import org.apache.usergrid.rest.test.resource.model.QueryParameters;
import org.apache.usergrid.rest.test.resource.model.Token;
+import org.apache.usergrid.corepersistence.index.ReIndexService.Status;
import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.junit.Ignore;
import org.junit.Test;
@@ -68,35 +69,51 @@ public class IndexResourceIT extends AbstractRestIT {
.get(clientSetup.getSuperuserName(),clientSetup.getSuperuserPassword());
QueryParameters queryParameters = new QueryParameters();
- queryParameters.addParam( "access_token",superUserToken.getAccessToken());
+ queryParameters.addParam("access_token", superUserToken.getAccessToken());
ApiResponse result = clientSetup.getRestClient()
- .pathResource( "system/index/rebuild/" + clientSetup.getAppUuid() + "/StOrElaTloNs" )
- .post( false, ApiResponse.class, null, queryParameters, true );
+ .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/StOrElaTloNs")
+ .post(false, ApiResponse.class, null, queryParameters, true);
assertNotNull(result);
+ assertEquals(Status.STARTED.name(), result.getStatus());
//try the reindex endpoint with all lowercase Characters
queryParameters = new QueryParameters();
- queryParameters.addParam( "access_token",clientSetup.getSuperuserToken().getAccessToken() );
+ queryParameters.addParam("access_token", superUserToken.getAccessToken());
result = clientSetup.getRestClient()
- .pathResource( "system/index/rebuild/"+clientSetup.getAppUuid()+"/storelatlons" )
- .post( false, ApiResponse.class,null,queryParameters,true);
- String status = result.getProperties().get("jobId").toString();
-
- assertNotNull( result );
+ .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/storelatlons")
+ .post(false, ApiResponse.class, null, queryParameters, true);
+
+ assertNotNull(result);
+ //at this point, this could return a result of the previous reindex, or if it has completed, it will create a new job
+ assertNotEquals(Status.UNKNOWN.name(), result.getStatus());
WebTarget res = clientSetup.getRestClient()
- .pathResource( "system/index/rebuild/" + result.getProperties()
- .get( "jobId" ).toString() )
+ .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/storelatlons")
.getTarget();
HttpAuthenticationFeature feature = HttpAuthenticationFeature.basicBuilder()
.credentials( "superuser", "superpassword" ).build();
result = res.register( feature ).request().get( ApiResponse.class );
-
- assertNotNull( result );
- assertEquals(status,result.getProperties().get("jobId").toString());
+ assertNotNull(result);
+
+ int retry = 0;
+ while(retry < 5 && !result.getStatus().equals(Status.COMPLETE.name())) {
+ try {
+ //hope reindex completes, if not, that's still ok
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+
+ }
+ result = res.register( feature ).request().get( ApiResponse.class );
+ retry++;
+ assertNotNull(result);
+ }
+
+ Map<String, Object> resultMap = result.getProperties();
+ assertNotNull( resultMap );
+ assertEquals(1,resultMap.get("numberQueued"));
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java b/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java
index 659520e..892a440 100644
--- a/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java
+++ b/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java
@@ -120,6 +120,7 @@ public class NewOrgAppAdminRule implements TestRule {
* Create the org admin and application
*/
protected void before( Description description ) throws Exception {
+ logger.info( "Test {}: Starting with application", description.getDisplayName() );
final String className = description.getClassName();
final String methodName = description.getMethodName();
final String uuidString = newUUIDString();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
index 6424dbd..d2932c2 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java
@@ -32,12 +32,14 @@ import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
+import net.jcip.annotations.NotThreadSafe;
+
import java.util.*;
import static org.apache.usergrid.TestHelper.*;
import static org.junit.Assert.*;
-
+@NotThreadSafe
public class OrganizationConfigIT {
@Rule
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index 870b678..2268ea6 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@ -59,6 +59,8 @@ import com.amazonaws.SDKGlobalConfiguration;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
+import net.jcip.annotations.NotThreadSafe;
+
import static org.apache.usergrid.TestHelper.newUUIDString;
import static org.apache.usergrid.TestHelper.uniqueApp;
import static org.apache.usergrid.TestHelper.uniqueOrg;
@@ -74,6 +76,7 @@ import static org.mockito.Mockito.when;
*
*
*/
+@NotThreadSafe
public class ExportServiceIT {
private static final Logger logger = LoggerFactory.getLogger( ExportServiceIT.class );