You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/12/01 15:25:31 UTC
[01/29] usergrid git commit: Add back in shard meta removal.
Repository: usergrid
Updated Branches:
refs/heads/USERGRID-872 c8ff1022d -> bfeb91b48
Add back in shard meta removal.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/afffdf27
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/afffdf27
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/afffdf27
Branch: refs/heads/USERGRID-872
Commit: afffdf278242ccb6e309cff220af371f32a0c0e4
Parents: 8ed1bc7
Author: Michael Russo <mi...@gmail.com>
Authored: Mon Nov 16 08:34:01 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Mon Nov 16 08:34:01 2015 -0800
----------------------------------------------------------------------
.../impl/shard/impl/ShardGroupDeletionImpl.java | 168 +++++++++----------
1 file changed, 84 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/afffdf27/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index 4d6be1b..d289f30 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -125,90 +125,90 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
final ShardEntryGroup shardEntryGroup,
final Iterator<MarkedEdge> edgeIterator ) {
- //TEMPORARILY Removed until further testing
- return DeleteResult.NO_OP;
-
-// logger.trace( "Beginning audit of shard group {}", shardEntryGroup );
-//
-// /**
-// * Compaction is pending, we cannot check it
-// */
-// if ( shardEntryGroup.isCompactionPending() ) {
-// logger.trace( "Shard group {} is compacting, not auditing group", shardEntryGroup );
-// return DeleteResult.COMPACTION_PENDING;
-// }
-//
-//
-// final long currentTime = timeService.getCurrentTime();
-//
-// if ( shardEntryGroup.isNew( currentTime ) ) {
-// logger.trace( "Shard group {} contains a shard that is is too new, not auditing group", shardEntryGroup );
-// return DeleteResult.TOO_NEW;
-// }
-//
-// /**
-// * We have edges, and therefore cannot delete them
-// */
-// if ( edgeIterator.hasNext() ) {
-// logger.trace( "Shard group {} has edges, not deleting", shardEntryGroup );
-//
-// return DeleteResult.CONTAINS_EDGES;
-// }
-//
-//
-// //now we can proceed based on the shard meta state and we don't have any edge
-//
-// DeleteResult result = DeleteResult.NO_OP;
-//
-// MutationBatch rollup = null;
-//
-// for ( final Shard shard : shardEntryGroup.getReadShards() ) {
-//
-// //skip the min shard
-// if(shard.isMinShard()){
-// logger.trace( "Shard {} in group {} is the minimum, not deleting", shard, shardEntryGroup );
-// continue;
-// }
-//
-// //The shard is not compacted, we cannot remove it. This should never happen, a bit of an "oh shit" scenario.
-// //the isCompactionPending should return false in this case
-// if(!shard.isCompacted()){
-// logger.warn( "Shard {} in group {} is not compacted yet was checked. Short circuiting", shard, shardEntryGroup );
-// return DeleteResult.NO_OP;
-// }
-//
-//
-// final MutationBatch shardRemovalMutation =
-// edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
-//
-// if ( rollup == null ) {
-// rollup = shardRemovalMutation;
-// }
-//
-// else {
-// rollup.mergeShallow( shardRemovalMutation );
-// }
-//
-// result = DeleteResult.DELETED;
-//
-// logger.trace( "Removing shard {} in group {}", shard, shardEntryGroup );
-// }
-//
-//
-// if( rollup != null) {
-//
-// try {
-// rollup.execute();
-// }
-// catch ( ConnectionException e ) {
-// logger.error( "Unable to execute shard deletion", e );
-// throw new RuntimeException( "Unable to execute shard deletion", e );
-// }
-// }
-//
-// logger.trace( "Completed auditing shard group {}", shardEntryGroup );
-//
-// return result;
+ //Use ths to TEMPORARILY remove deletes from occurring
+ //return DeleteResult.NO_OP;
+
+ logger.trace( "Beginning audit of shard group {}", shardEntryGroup );
+
+ /**
+ * Compaction is pending, we cannot check it
+ */
+ if ( shardEntryGroup.isCompactionPending() ) {
+ logger.trace( "Shard group {} is compacting, not auditing group", shardEntryGroup );
+ return DeleteResult.COMPACTION_PENDING;
+ }
+
+
+ final long currentTime = timeService.getCurrentTime();
+
+ if ( shardEntryGroup.isNew( currentTime ) ) {
+ logger.trace( "Shard group {} contains a shard that is is too new, not auditing group", shardEntryGroup );
+ return DeleteResult.TOO_NEW;
+ }
+
+ /**
+ * We have edges, and therefore cannot delete them
+ */
+ if ( edgeIterator.hasNext() ) {
+ logger.trace( "Shard group {} has edges, not deleting", shardEntryGroup );
+
+ return DeleteResult.CONTAINS_EDGES;
+ }
+
+
+ //now we can proceed based on the shard meta state and we don't have any edge
+
+ DeleteResult result = DeleteResult.NO_OP;
+
+ MutationBatch rollup = null;
+
+ for ( final Shard shard : shardEntryGroup.getReadShards() ) {
+
+ //skip the min shard
+ if(shard.isMinShard()){
+ logger.trace( "Shard {} in group {} is the minimum, not deleting", shard, shardEntryGroup );
+ continue;
+ }
+
+ //The shard is not compacted, we cannot remove it. This should never happen, a bit of an "oh shit" scenario.
+ //the isCompactionPending should return false in this case
+ if(!shard.isCompacted()){
+ logger.warn( "Shard {} in group {} is not compacted yet was checked. Short circuiting", shard, shardEntryGroup );
+ return DeleteResult.NO_OP;
+ }
+
+
+ final MutationBatch shardRemovalMutation =
+ edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
+
+ if ( rollup == null ) {
+ rollup = shardRemovalMutation;
+ }
+
+ else {
+ rollup.mergeShallow( shardRemovalMutation );
+ }
+
+ result = DeleteResult.DELETED;
+
+ logger.info( "Removing shard {} in group {}", shard, shardEntryGroup );
+ }
+
+
+ if( rollup != null) {
+
+ try {
+ rollup.execute();
+ }
+ catch ( ConnectionException e ) {
+ logger.error( "Unable to execute shard deletion", e );
+ throw new RuntimeException( "Unable to execute shard deletion", e );
+ }
+ }
+
+ logger.trace( "Completed auditing shard group {}", shardEntryGroup );
+
+ return result;
}
[27/29] usergrid git commit: Test fix.
Posted by sn...@apache.org.
Test fix.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ee4d504a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ee4d504a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ee4d504a
Branch: refs/heads/USERGRID-872
Commit: ee4d504a8e9a68913e399067fbbced238a1032e9
Parents: 5dfdf81
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Nov 23 16:11:53 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Nov 23 16:11:53 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/usergrid/rest/management/AdminUsersIT.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ee4d504a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
index b346c28..cf27644 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
@@ -478,7 +478,8 @@ public class AdminUsersIT extends AbstractRestIT {
fail("Password reset request should have failed");
} catch ( BadRequestException expected ) {
- assertTrue( expected.getResponse().getEntity().toString().contains( "Could not find" ) );
+ String body = expected.getResponse().readEntity( String.class );
+ assertTrue( body.contains( "Could not find" ) );
}
String html = management().users().user( clientSetup.getUsername() ).resetpw().getTarget().request()
[15/29] usergrid git commit: Don't filter out properties that have
name of 'size'. The internal size prop is noted in our metadata object,
not at the root fields level.
Posted by sn...@apache.org.
Don't filter out properties that have name of 'size'. The internal size prop is noted in our metadata object, not at the root fields level.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/aea92641
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/aea92641
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/aea92641
Branch: refs/heads/USERGRID-872
Commit: aea926411c29e57e6d65cd50cbacf921821b16e1
Parents: faa234a
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 11:07:55 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 11:07:55 2015 -0800
----------------------------------------------------------------------
.../usergrid/persistence/model/entity/MapToEntityConverter.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/aea92641/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
index 2357a57..a225820 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
@@ -50,9 +50,7 @@ public class MapToEntityConverter{
public Entity fromMap(final Entity entity,final Map<String, Object> map,final SchemaManager schemaManager, final String entityType, boolean topLevel) {
for ( String fieldName : map.keySet() ) {
- if(fieldName.equals("size")){
- continue;
- }
+
Object value = map.get( fieldName );
boolean unique = schemaManager == null ? topLevel : topLevel && schemaManager.isPropertyUnique(entityType, fieldName);
//cannot store fields that aren't locations
[22/29] usergrid git commit: Re-enable 6 tests that were ignored or
commented out, add JIRA issue for the one that is still ignored.
Posted by sn...@apache.org.
Re-enable 6 tests that were ignored or commented out, add JIRA issue for the one that is still ignored.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cd1675c8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cd1675c8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cd1675c8
Branch: refs/heads/USERGRID-872
Commit: cd1675c833a8fc29b3169916b9e4808a7144cc02
Parents: 6fffe93
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Nov 23 12:26:50 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Nov 23 12:26:50 2015 -0500
----------------------------------------------------------------------
.../usergrid/rest/management/AdminUsersIT.java | 97 ++++++++++++++------
1 file changed, 71 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cd1675c8/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
index 09a76aa..a022f80 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.rest.management;
+import net.jcip.annotations.NotThreadSafe;
import org.apache.usergrid.management.MockImapClient;
import org.apache.usergrid.persistence.core.util.StringUtils;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
@@ -36,10 +37,7 @@ import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.core.Form;
import javax.ws.rs.core.Response;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import static org.apache.usergrid.management.AccountCreationProps.*;
import static org.junit.Assert.*;
@@ -48,6 +46,7 @@ import static org.junit.Assert.*;
/**
* Contains all tests relating to Admin Users
*/
+@NotThreadSafe
public class AdminUsersIT extends AbstractRestIT {
ManagementResource management;
@@ -187,7 +186,6 @@ public class AdminUsersIT extends AbstractRestIT {
* TODO:test for parallel test that changing the properties here won't affect other tests
* @throws Exception
*/
- @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1115. breaks other tests")
@Test
public void testUnconfirmedAdminLogin() throws Exception{
@@ -273,7 +271,6 @@ public class AdminUsersIT extends AbstractRestIT {
* Test that the system admin doesn't need a confirmation email
* @throws Exception
*/
- @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1115. breaks other tests")
@Test
public void testSystemAdminNeedsNoConfirmation() throws Exception{
//Save original properties to return them to normal at the end of the test
@@ -311,8 +308,6 @@ public class AdminUsersIT extends AbstractRestIT {
* Test that the test account doesn't need confirmation and is created automatically.
* @throws Exception
*/
- @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1115. Test account problem")
- // Test doesn't pass because the test account isn't getting correct instantiated
@Test
public void testTestUserNeedsNoConfirmation() throws Exception{
//Save original properties to return them to normal at the end of the test
@@ -348,9 +343,7 @@ public class AdminUsersIT extends AbstractRestIT {
* Update the current management user and make sure the change persists
* @throws Exception
*/
- @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1115.")
- // Fails because we cannot GET a management user with a super user token - only with an Admin level token.
- // But, we can PUT with a superuser token. This test will work once that issue has been resolved.
+ @Ignore("Pending new feature https://issues.apache.org/jira/browse/USERGRID-1127")
@Test
public void updateManagementUser() throws Exception {
@@ -400,25 +393,32 @@ public class AdminUsersIT extends AbstractRestIT {
assertFalse( inbox.isEmpty() );
}
- @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1115. Viewables issue.")
- // Test is broken due to viewables not being properly returned in the embedded tomcat
+
@Test
public void checkFormPasswordReset() throws Exception {
-
+ // initiate password reset
management().users().user( clientSetup.getUsername() ).resetpw().post(new Form());
+ refreshIndex();
- //Create mocked inbox
+ // create mocked inbox, get password reset email and extract token
List<Message> inbox = Mailbox.get( clientSetup.getEmail() );
assertFalse( inbox.isEmpty() );
MockImapClient client = new MockImapClient( "mockserver.com", "test-user-46", "somepassword" );
client.processMail();
- //Get email with confirmation token and extract token
- Message confirmation = inbox.get( 0 );
- assertEquals( "User Account Confirmation: " + clientSetup.getEmail(), confirmation.getSubject() );
- String token = getTokenFromMessage( confirmation );
+ // Get email with confirmation token and extract token
+
+ String token = null;
+ Iterator<Message> msgIterator = inbox.iterator();
+ while ( token == null && msgIterator.hasNext() ) {
+ Message msg = msgIterator.next();
+ if ( msg.getSubject().equals("Password Reset") ) {
+ token = getTokenFromMessage( msg );
+ }
+ }
+ assertNotNull( token );
Form formData = new Form();
formData.param( "token", token );
@@ -438,6 +438,7 @@ public class AdminUsersIT extends AbstractRestIT {
assertTrue( html.contains( "invalid token" ) );
}
+
// @Test
// public void passwordResetIncorrectUserName() throws Exception {
//
@@ -519,9 +520,54 @@ public class AdminUsersIT extends AbstractRestIT {
}
//TODO: won't work until resetpw viewables are fixed in the embedded environment.
-// @Test
-// public void checkPasswordChangeTime() throws Exception {
-//
+ @Test
+ public void checkPasswordChangeTime() throws Exception {
+
+ // request password reset
+
+ management().users().user( clientSetup.getUsername() ).resetpw().post(new Form());
+ refreshIndex();
+
+ // get resetpw token from email
+
+ List<Message> inbox = Mailbox.get( clientSetup.getEmail() );
+ assertFalse( inbox.isEmpty() );
+ MockImapClient client = new MockImapClient( "mockserver.com", "test-user-46", "somepassword" );
+ client.processMail();
+ String token = null;
+ Iterator<Message> msgIterator = inbox.iterator();
+ while ( token == null && msgIterator.hasNext() ) {
+ Message msg = msgIterator.next();
+ if ( msg.getSubject().equals("Password Reset") ) {
+ token = getTokenFromMessage( msg );
+ }
+ }
+ assertNotNull( token );
+
+ // reset the password to sesame
+
+ Form formData = new Form();
+ formData.param( "token", token );
+ formData.param( "password1", "sesame" );
+ formData.param( "password2", "sesame" );
+ String html = management().users().user( clientSetup.getUsername() ).resetpw().getTarget().request()
+ .post( javax.ws.rs.client.Entity.form(formData), String.class );
+ assertTrue( html.contains( "password set" ) );
+
+
+
+
+ // login with new password and get token
+
+ // check password changed time
+
+ // change password again by posting JSON
+
+ // get password and check password change time again
+
+ // login via /me end-point and check password change time again
+
+
// final TestUser user = context.getActiveUser();
// String email = user.getEmail();
// UserInfo userInfo = setup.getMgmtSvc().getAdminUserByEmail( email );
@@ -573,9 +619,9 @@ public class AdminUsersIT extends AbstractRestIT {
//
// Long changeTime3 = node.get( "passwordChanged" ).longValue();
// assertEquals( changeTime2, changeTime3 );
-// }
-//
-//
+ }
+
+
/**
@@ -638,7 +684,6 @@ public class AdminUsersIT extends AbstractRestIT {
}
}
- @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1115. breaks other tests")
@Test
public void testProperties(){
ApiResponse originalTestPropertiesResponse = clientSetup.getRestClient().testPropertiesResource().get();
[02/29] usergrid git commit: Added test to prove shard ordering.
Posted by sn...@apache.org.
Added test to prove shard ordering.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2b2793ff
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2b2793ff
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2b2793ff
Branch: refs/heads/USERGRID-872
Commit: 2b2793ffeddb0836ced383b13c25d8f86432b632
Parents: afffdf2
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 16 19:00:20 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 16 19:00:20 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/ShardTest.java | 65 ++++++++++++++++++++
1 file changed, 65 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2b2793ff/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardTest.java
new file mode 100644
index 0000000..478873f
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard;
+
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+
+public class ShardTest {
+
+ @Test
+ public void testShardIndexOrder() {
+ final Shard first = new Shard( 1, 1, false );
+
+
+ final Shard second = new Shard( 2, 1, false );
+
+ assertTrue( first.compareTo( second ) < 0 );
+
+ assertTrue( second.compareTo( first ) > 0 );
+ }
+
+
+ @Test
+ public void testShardTimestapOrder() {
+ final Shard first = new Shard( 1, 1, false );
+
+
+ final Shard second = new Shard( 1, 2, false );
+
+ assertTrue( first.compareTo( second ) < 0 );
+
+ assertTrue( second.compareTo( first ) > 0 );
+ }
+
+
+ @Test
+ public void testShardCompactedOrder() {
+ final Shard first = new Shard( 1, 1, false );
+
+
+ final Shard second = new Shard( 1, 1, true );
+
+ assertTrue( first.compareTo( second ) < 0 );
+
+ assertTrue( second.compareTo( first ) > 0 );
+ }
+}
[16/29] usergrid git commit: Use UUID compare for more precision.
Posted by sn...@apache.org.
Use UUID compare for more precision.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b1ed8b6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b1ed8b6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b1ed8b6
Branch: refs/heads/USERGRID-872
Commit: 1b1ed8b63bbfd3c5fcf6f0e4a09f2009e8709d53
Parents: aea9264
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 12:21:48 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 12:21:48 2015 -0800
----------------------------------------------------------------------
.../usergrid/corepersistence/asyncevents/EventBuilderImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b1ed8b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index f7f7ba3..c9ae6da 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.util.List;
+import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -144,7 +145,7 @@ public class EventBuilderImpl implements EventBuilder {
ecmDeleteObservable =
ecm.getVersions( entityId )
- .filter( mvccLogEntry-> mvccLogEntry.getVersion().timestamp() <= mostRecentlyMarked.getVersion().timestamp())
+ .filter( mvccLogEntry-> UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) < 0)
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
[28/29] usergrid git commit: Merge branch 'master' into USERGRID-872
Posted by sn...@apache.org.
Merge branch 'master' into USERGRID-872
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/668a8012
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/668a8012
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/668a8012
Branch: refs/heads/USERGRID-872
Commit: 668a80127001d917482e035bb999bea2a8f41da1
Parents: c8ff102 ee4d504
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Nov 24 16:27:02 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Nov 24 16:27:02 2015 -0500
----------------------------------------------------------------------
.../asyncevents/EventBuilderImpl.java | 47 ++--
.../usergrid/persistence/graph/GraphFig.java | 2 +-
.../impl/EdgeSerializationImpl.java | 61 ++++-
.../shard/impl/EdgeShardSerializationImpl.java | 3 +-
.../shard/impl/NodeShardAllocationImpl.java | 11 +-
.../shard/impl/ShardGroupColumnIterator.java | 4 +-
.../shard/impl/ShardGroupCompactionImpl.java | 5 +-
.../impl/shard/impl/ShardGroupDeletionImpl.java | 174 +++++++------
.../graph/GraphManagerShardConsistencyIT.java | 40 +--
.../impl/shard/EdgeShardSerializationTest.java | 44 ++++
.../impl/shard/NodeShardAllocationTest.java | 6 +-
.../impl/shard/ShardEntryGroupTest.java | 31 +++
.../serialization/impl/shard/ShardTest.java | 65 +++++
.../model/entity/MapToEntityConverter.java | 7 +-
.../applications/ApplicationResourceIT.java | 33 ++-
.../collection/groups/GroupResourceIT.java | 115 ++++++---
.../usergrid/rest/management/AdminUsersIT.java | 254 +++++++++++--------
17 files changed, 597 insertions(+), 305 deletions(-)
----------------------------------------------------------------------
[19/29] usergrid git commit: formatting change only. This closes #442
Posted by sn...@apache.org.
formatting change only. This closes #442
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9bfbc531
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9bfbc531
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9bfbc531
Branch: refs/heads/USERGRID-872
Commit: 9bfbc531c4e70eedaff435033512e0d48f8df25c
Parents: e5b9730
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Nov 19 15:34:44 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Nov 19 15:34:44 2015 -0500
----------------------------------------------------------------------
.../usergrid/persistence/model/entity/MapToEntityConverter.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bfbc531/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
index a225820..aff7439 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
@@ -47,7 +47,8 @@ public class MapToEntityConverter{
return fromMap( entity, map, null, null, topLevel );
}
- public Entity fromMap(final Entity entity,final Map<String, Object> map,final SchemaManager schemaManager, final String entityType, boolean topLevel) {
+ public Entity fromMap(final Entity entity,final Map<String, Object> map,final
+ SchemaManager schemaManager, final String entityType, boolean topLevel) {
for ( String fieldName : map.keySet() ) {
[05/29] usergrid git commit: Added logging to deletion task.
Posted by sn...@apache.org.
Added logging to deletion task.
Updated consistency test to have 2 read workers.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c44dfdab
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c44dfdab
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c44dfdab
Branch: refs/heads/USERGRID-872
Commit: c44dfdabfc23db4e0d2eed739d19f865c58288cd
Parents: ad12f35
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 09:07:34 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 09:07:34 2015 -0700
----------------------------------------------------------------------
.../impl/shard/impl/ShardGroupDeletionImpl.java | 9 +++--
.../graph/GraphManagerShardConsistencyIT.java | 40 +++++++++++---------
2 files changed, 28 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c44dfdab/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index c460a5d..373d42b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -138,6 +138,7 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
return DeleteResult.COMPACTION_PENDING;
}
+ logger.trace( "Shard group {} has no compaction pending", shardEntryGroup );
final long currentTime = timeService.getCurrentTime();
@@ -146,6 +147,8 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
return DeleteResult.TOO_NEW;
}
+ logger.trace( "Shard group {} has passed the delta timeout at {}", shardEntryGroup, currentTime );
+
/**
* We have edges, and therefore cannot delete them
*/
@@ -156,6 +159,9 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
}
+ logger.trace( "Shard group {} has no edges continuing to delete", shardEntryGroup, currentTime );
+
+
//now we can proceed based on the shard meta state and we don't have any edge
DeleteResult result = DeleteResult.NO_OP;
@@ -178,9 +184,6 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
}
- logger.info( "Deleting shard {} at time {} ", shard, timeService.getCurrentTime() );
-
-
final MutationBatch shardRemovalMutation =
edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c44dfdab/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index a9f9cb7..b903c6c 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -266,31 +266,35 @@ public class GraphManagerShardConsistencyIT {
final Meter readMeter = registry.meter( "readThroughput" );
- /**
- * Start reading continuously while we migrate data to ensure our view is always correct
- */
- final ListenableFuture<Long> future =
- executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
-
final List<Throwable> failures = new ArrayList<>();
+ for(int i = 0; i < 2; i ++) {
- //add the future
- Futures.addCallback( future, new FutureCallback<Long>() {
- @Override
- public void onSuccess( @Nullable final Long result ) {
- log.info( "Successfully ran the read, re-running" );
+ /**
+ * Start reading continuously while we migrate data to ensure our view is always correct
+ */
+ final ListenableFuture<Long> future =
executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
- }
- @Override
- public void onFailure( final Throwable t ) {
- failures.add( t );
- log.error( "Failed test!", t );
- }
- } );
+ //add the future
+ Futures.addCallback( future, new FutureCallback<Long>() {
+
+ @Override
+ public void onSuccess( @Nullable final Long result ) {
+ log.info( "Successfully ran the read, re-running" );
+ executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
+ }
+
+
+ @Override
+ public void onFailure( final Throwable t ) {
+ failures.add( t );
+ log.error( "Failed test!", t );
+ }
+ } );
+ }
int compactedCount;
[23/29] usergrid git commit: Re-enable 6 tests that were ignored or
commented out, add JIRA issue for the one that is still ignored.
Posted by sn...@apache.org.
Re-enable 6 tests that were ignored or commented out, add JIRA issue for the one that is still ignored.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/46f9802e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/46f9802e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/46f9802e
Branch: refs/heads/USERGRID-872
Commit: 46f9802e2fe9d8d36ecbfc4b0b86ae4756e8e03d
Parents: cd1675c
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Nov 23 12:27:04 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Nov 23 12:27:04 2015 -0500
----------------------------------------------------------------------
.../usergrid/rest/management/AdminUsersIT.java | 162 ++++++++++---------
1 file changed, 82 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/46f9802e/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
index a022f80..b346c28 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/AdminUsersIT.java
@@ -32,6 +32,7 @@ import org.jvnet.mock_javamail.Mailbox;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.internet.MimeMultipart;
+import javax.ws.rs.BadRequestException;
import javax.ws.rs.ClientErrorException;
import javax.ws.rs.InternalServerErrorException;
import javax.ws.rs.core.Form;
@@ -439,31 +440,52 @@ public class AdminUsersIT extends AbstractRestIT {
}
-// @Test
-// public void passwordResetIncorrectUserName() throws Exception {
-//
-// String email = "test2@usergrid.com";
-// setup.getMgmtSvc().createAdminUser( "test2", "test2", "test2@usergrid.com", "sesa2me", false, false );
-// UserInfo userInfo = setup.getMgmtSvc().getAdminUserByEmail( email );
-// String resetToken = setup.getMgmtSvc().getPasswordResetTokenForAdminUser( userInfo.getUuid(), 15000 );
-//
-// assertTrue( setup.getMgmtSvc().checkPasswordResetTokenForAdminUser( userInfo.getUuid(), resetToken ) );
-//
-// Form formData = new Form();
-// formData.add( "token", resetToken );
-// formData.add( "password1", "sesa2me" );
-// formData.add( "password2", "sesa2me" );
-//
-// String html = resource().path( "/management/users/" + "noodle" + userInfo.getUsername() + "/resetpw" )
-// .type( MediaType.APPLICATION_FORM_URLENCODED_TYPE ).post( String.class, formData );
-//
-// assertTrue( html.contains( "Incorrect username entered" ) );
-//
-// html = resource().path( "/management/users/" + userInfo.getUsername() + "/resetpw" )
-// .type( MediaType.APPLICATION_FORM_URLENCODED_TYPE ).post( String.class, formData );
-//
-// assertTrue( html.contains( "password set" ) );
-// }
+ @Test
+ public void passwordResetIncorrectUserName() throws Exception {
+
+ // initiate password reset
+ management().users().user( clientSetup.getUsername() ).resetpw().post(new Form());
+ refreshIndex();
+
+ // create mocked inbox, get password reset email and extract token
+ List<Message> inbox = Mailbox.get( clientSetup.getEmail() );
+ assertFalse( inbox.isEmpty() );
+
+ MockImapClient client = new MockImapClient( "mockserver.com", "test-user-47", "somepassword" );
+ client.processMail();
+
+ // Get email with confirmation token and extract token
+ String token = null;
+ Iterator<Message> msgIterator = inbox.iterator();
+ while ( token == null && msgIterator.hasNext() ) {
+ Message msg = msgIterator.next();
+ if ( msg.getSubject().equals("Password Reset") ) {
+ token = getTokenFromMessage( msg );
+ }
+ }
+ assertNotNull( token );
+
+ Form formData = new Form();
+ formData.param( "token", token );
+ formData.param( "password1", "sesame" );
+ formData.param( "password2", "sesame" );
+
+ try {
+
+ String html = management().users().user( "mr_fatfinger" ).resetpw().getTarget().request()
+ .post( javax.ws.rs.client.Entity.form( formData ), String.class );
+
+ fail("Password reset request should have failed");
+
+ } catch ( BadRequestException expected ) {
+ assertTrue( expected.getResponse().getEntity().toString().contains( "Could not find" ) );
+ }
+
+ String html = management().users().user( clientSetup.getUsername() ).resetpw().getTarget().request()
+ .post( javax.ws.rs.client.Entity.form(formData), String.class );
+
+ assertTrue( html.contains( "password set" ) );
+ }
/**
@@ -519,7 +541,6 @@ public class AdminUsersIT extends AbstractRestIT {
}
}
- //TODO: won't work until resetpw viewables are fixed in the embedded environment.
@Test
public void checkPasswordChangeTime() throws Exception {
@@ -553,72 +574,53 @@ public class AdminUsersIT extends AbstractRestIT {
String html = management().users().user( clientSetup.getUsername() ).resetpw().getTarget().request()
.post( javax.ws.rs.client.Entity.form(formData), String.class );
assertTrue( html.contains( "password set" ) );
-
-
-
+ refreshIndex();
// login with new password and get token
+ Map<String, Object> payload = new HashMap<String, Object>() {{
+ put("grant_type", "password");
+ put("username", clientSetup.getUsername() );
+ put("password", "sesame");
+ }};
+ ApiResponse response = management().token().post( false, payload, null );
+
// check password changed time
+ Long changeTime = Long.parseLong( response.getProperties().get( "passwordChanged" ).toString() );
+ assertTrue( System.currentTimeMillis() - changeTime < 2000 );
+
// change password again by posting JSON
+ payload = new HashMap<String, Object>() {{
+ put("oldpassword", "sesame");
+ put("newpassword", "test");
+ }};
+ management().users().user( clientSetup.getUsername() ).password().post( false, payload, null );
+ refreshIndex();
+
// get password and check password change time again
+ payload = new HashMap<String, Object>() {{
+ put("grant_type", "password");
+ put("username", clientSetup.getUsername() );
+ put("password", "test");
+ }};
+ response = management().token().post( false, payload, null );
+
+ Long changeTime2 = Long.parseLong( response.getProperties().get( "passwordChanged" ).toString() );
+ assertTrue( changeTime < changeTime2 );
+ assertTrue( System.currentTimeMillis() - changeTime2 < 2000 );
+
// login via /me end-point and check password change time again
+ response = management().me().get( ApiResponse.class, new QueryParameters()
+ .addParam( "grant_type", "password" )
+ .addParam( "username", clientSetup.getUsername() )
+ .addParam( "password", "test" ) );
-// final TestUser user = context.getActiveUser();
-// String email = user.getEmail();
-// UserInfo userInfo = setup.getMgmtSvc().getAdminUserByEmail( email );
-// String resetToken = setup.getMgmtSvc().getPasswordResetTokenForAdminUser( userInfo.getUuid(), 15000 );
-//
-// refreshIndex(context.getOrgName(), context.getAppName());
-//
-// Form formData = new Form();
-// formData.add( "token", resetToken );
-// formData.add( "password1", "sesame" );
-// formData.add( "password2", "sesame" );
-//
-// String html = resource().path( "/management/users/" + userInfo.getUsername() + "/resetpw" )
-// .type( MediaType.APPLICATION_FORM_URLENCODED_TYPE ).post( String.class, formData );
-// assertTrue( html.contains( "password set" ) );
-//
-// refreshIndex(context.getOrgName(), context.getAppName());
-//
-// JsonNode node = mapper.readTree( resource().path( "/management/token" )
-// .queryParam( "grant_type", "password" )
-// .queryParam( "username", email ).queryParam( "password", "sesame" )
-// .accept( MediaType.APPLICATION_JSON )
-// .get( String.class ));
-//
-// Long changeTime = node.get( "passwordChanged" ).longValue();
-// assertTrue( System.currentTimeMillis() - changeTime < 2000 );
-//
-// Map<String, String> payload = hashMap( "oldpassword", "sesame" ).map( "newpassword", "test" );
-// node = mapper.readTree( resource().path( "/management/users/" + userInfo.getUsername() + "/password" )
-// .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-// .post( String.class, payload ));
-//
-// refreshIndex(context.getOrgName(), context.getAppName());
-//
-// node = mapper.readTree( resource().path( "/management/token" )
-// .queryParam( "grant_type", "password" )
-// .queryParam( "username", email )
-// .queryParam( "password", "test" )
-// .accept( MediaType.APPLICATION_JSON )
-// .get( String.class ));
-//
-// Long changeTime2 = node.get( "passwordChanged" ).longValue();
-// assertTrue( changeTime < changeTime2 );
-// assertTrue( System.currentTimeMillis() - changeTime2 < 2000 );
-//
-// node = mapper.readTree( resource().path( "/management/me" ).queryParam( "grant_type", "password" )
-// .queryParam( "username", email ).queryParam( "password", "test" ).accept( MediaType.APPLICATION_JSON )
-// .get( String.class ));
-//
-// Long changeTime3 = node.get( "passwordChanged" ).longValue();
-// assertEquals( changeTime2, changeTime3 );
+ Long changeTime3 = Long.parseLong( response.getProperties().get( "passwordChanged" ).toString() );
+ assertEquals( changeTime2, changeTime3 );
}
[21/29] usergrid git commit: Fix for USERGRID-1106: fixed by
rewriting test to do exactly what it used to do before the new framework
conversion.
Posted by sn...@apache.org.
Fix for USERGRID-1106: fixed by rewriting test to do exactly what it used to do before the new framework conversion.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c1f5e16a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c1f5e16a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c1f5e16a
Branch: refs/heads/USERGRID-872
Commit: c1f5e16a72bda590be1e890be005a636fd247550
Parents: 6fffe93
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Nov 20 09:56:30 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Nov 20 09:56:30 2015 -0500
----------------------------------------------------------------------
.../applications/ApplicationResourceIT.java | 33 ++++++++++++--------
1 file changed, 20 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c1f5e16a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
index 8d29290..e822f66 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
@@ -645,34 +645,41 @@ public class ApplicationResourceIT extends AbstractRestIT {
}
/**
- * Retrieve an access token using HTTP Basic authentication
+ * Retrieve an app user access token using HTTP Basic authentication
*/
@Test
- @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1106")
- //Are we trying to generate token with token? Couldn't find endpoint that accepts token for generating token
public void clientCredentialsFlowWithHeaderAuthorization() throws Exception {
- //retrieve the credentials
- Credentials orgCredentials = getAppCredentials();
- String clientId = orgCredentials.getClientId();
- String clientSecret = orgCredentials.getClientSecret();
+ // get app credentials from /<org>/<app>/credentials end-point (using admin credentials)
+ Credentials appCredentials = getAppCredentials();
+ String clientId = appCredentials.getClientId();
+ String clientSecret = appCredentials.getClientSecret();
+
+ // use app credentials to admin user access token
Token token = clientSetup.getRestClient().management().token()
.post(Token.class,new Token("client_credentials", clientId, clientSecret));
- //GET the token endpoint, adding authorization header
- Token apiResponse = this.app().token().getTarget( false )
+ String clientCredentials = clientId + ":" + clientSecret;
+ String encodedToken = Base64.encodeToString( clientCredentials.getBytes() );
+
+ Map<String, String> payload = hashMap( "grant_type", "client_credentials" );
+
+ // use admin user access token to get app user access token
+ Token apiResponse = this.app().token().getTarget( false ).request()
//add the auth header
- .request()
- .header( "Authorization", "Bearer " + token.getAccessToken() )
+ .header( "Authorization", "Basic " + encodedToken )
.accept( MediaType.APPLICATION_JSON )
- .post(javax.ws.rs.client.Entity.entity(
- hashMap( "grant_type", "client_credentials" ), MediaType.APPLICATION_JSON_TYPE ), Token.class );
+ .post(javax.ws.rs.client.Entity.entity(payload, MediaType.APPLICATION_JSON_TYPE ), Token.class );
//Assert that a valid token with a valid TTL is returned
assertNotNull("A valid response was returned.", apiResponse);
assertNull("There is no error.", apiResponse.getError());
assertNotNull("It has access_token.", apiResponse.getAccessToken());
assertNotNull("It has expires_in.", apiResponse.getExpirationDate());
+
+
+
+
}
/**
[10/29] usergrid git commit: Fixes incorrect default property
Posted by sn...@apache.org.
Fixes incorrect default property
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/784fe51c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/784fe51c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/784fe51c
Branch: refs/heads/USERGRID-872
Commit: 784fe51ca9593f813294865e198d1fd07b796aee
Parents: 8ba0ef0
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 11:27:27 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 11:27:27 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/persistence/graph/GraphFig.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/784fe51c/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
index ecb5b19..5968097 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
@@ -118,7 +118,7 @@ public interface GraphFig extends GuicyFig {
@Key(SHARD_CACHE_TIMEOUT)
long getShardCacheTimeout();
- @Default("60000")
+ @Default("90000")
@Key(SHARD_MIN_DELTA)
long getShardMinDelta();
[20/29] usergrid git commit: Merge branch 'release' of
https://git-wip-us.apache.org/repos/asf/usergrid
Posted by sn...@apache.org.
Merge branch 'release' of https://git-wip-us.apache.org/repos/asf/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6fffe936
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6fffe936
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6fffe936
Branch: refs/heads/USERGRID-872
Commit: 6fffe936bbdfa188892e8e8a9190eb03ec9c17a5
Parents: 6f3541b 9bfbc53
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 20:25:43 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 20:25:43 2015 -0800
----------------------------------------------------------------------
.../asyncevents/EventBuilderImpl.java | 47 +++--
.../usergrid/persistence/graph/GraphFig.java | 2 +-
.../impl/EdgeSerializationImpl.java | 61 ++++++-
.../shard/impl/EdgeShardSerializationImpl.java | 3 +-
.../shard/impl/NodeShardAllocationImpl.java | 11 +-
.../shard/impl/ShardGroupColumnIterator.java | 4 +-
.../shard/impl/ShardGroupCompactionImpl.java | 5 +-
.../impl/shard/impl/ShardGroupDeletionImpl.java | 174 ++++++++++---------
.../graph/GraphManagerShardConsistencyIT.java | 40 +++--
.../impl/shard/EdgeShardSerializationTest.java | 44 +++++
.../impl/shard/NodeShardAllocationTest.java | 6 +-
.../impl/shard/ShardEntryGroupTest.java | 31 ++++
.../serialization/impl/shard/ShardTest.java | 65 +++++++
.../model/entity/MapToEntityConverter.java | 7 +-
14 files changed, 351 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6fffe936/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
[12/29] usergrid git commit: Fix for USERGRID-1119 which tests the
private group activities use case.
Posted by sn...@apache.org.
Fix for USERGRID-1119 which tests the private group activities use case.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/28a627a5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/28a627a5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/28a627a5
Branch: refs/heads/USERGRID-872
Commit: 28a627a56d40bc192b92071b91a65564113b778c
Parents: 6f3541b
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Nov 19 12:35:47 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Nov 19 12:35:47 2015 -0500
----------------------------------------------------------------------
.../collection/groups/GroupResourceIT.java | 115 ++++++++++++-------
1 file changed, 75 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/28a627a5/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java
index bd75adf..769852d 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/groups/GroupResourceIT.java
@@ -25,6 +25,7 @@ import org.junit.Ignore;
import org.junit.Test;
import javax.ws.rs.ClientErrorException;
+import javax.ws.rs.NotAuthorizedException;
import java.io.IOException;
import static org.junit.Assert.*;
@@ -388,21 +389,19 @@ public class GroupResourceIT extends AbstractRestIT {
}
- /***
- *
- * Post a group activity and make sure it can be read back only by group members
- *
+ /**
+ * Post a group activities and test that the appear in the groups activities feed, and that
+ * they appear only in the personal activity feeds of the group's members.
*/
- @Ignore("Pending https://issues.apache.org/jira/browse/USERGRID-1119. Fails. See todo in the test itself.")
@Test
- public void postGroupActivity() throws IOException {
-
+ public void postPrivateGroupActivity() throws IOException {
//1. create a group
String groupName = "testgroup";
String groupPath = "testgroup";
Entity group = this.createGroup(groupName, groupPath);
+
//2. create user 1
String user1Username = "fred";
String user1Email = "fred@usergrid.com";
@@ -421,21 +420,51 @@ public class GroupResourceIT extends AbstractRestIT {
password = "password";
Entity user3 = this.createUser(user3Username, user3Email, password);
+
//5. add user1 to the group
- Entity addUser1Response = this.app().collection("users").entity(user1).connection().collection("groups").entity(group).post();
+ Entity addUser1Response = this.app().collection("users")
+ .entity(user1).connection().collection("groups").entity(group).post();
assertEquals(addUser1Response.get("name"), groupName);
//6. add user2 to the group
- Entity addUser2Response = this.app().collection("users").entity(user2).connection().collection("groups").entity(group).post();
+ Entity addUser2Response = this.app().collection("users")
+ .entity(user2).connection().collection("groups").entity(group).post();
assertEquals(addUser2Response.get("name"), groupName);
+
// user 3 does not get added to the group
+
//7. get all the users in the groups
this.refreshIndex();
Collection usersInGroup = this.app().collection("groups").uniqueID(groupName).connection("users").get();
assertEquals(usersInGroup.getResponse().getEntityCount(), 2);
+ //7a. delete default role and make group private, only accessible to those with "group1role"
+ this.app().collection("role").uniqueID("Default").delete();
+ Entity data = new Entity().chainPut("name", "group1role");
+ this.app().collection("roles").post(data);
+ this.refreshIndex();
+
+ Entity perms = new Entity();
+ String permission = "get,post,put,delete:/groups/" + group.getUuid() + "/**";
+ perms.put("permission",permission);
+ this.app().collection("roles").uniqueID("group1role").connection("permissions").post(perms);
+ this.app().collection("roles").uniqueID("group1role").connection("users").uniqueID( user1Username ).post();
+ this.app().collection("roles").uniqueID("group1role").connection("users").uniqueID( user2Username ).post();
+ this.refreshIndex();
+
+ //7b. everybody gets access to /activities
+ perms = new Entity();
+ permission = "get:/activities/**";
+ perms.put("permission",permission);
+ this.app().collection("roles").uniqueID("Guest").connection("permissions").post(perms);
+ this.refreshIndex();
+ this.app().collection("roles").uniqueID("Guest").connection("users").uniqueID( user1Username ).post();
+ this.app().collection("roles").uniqueID("Guest").connection("users").uniqueID( user2Username ).post();
+ this.app().collection("roles").uniqueID("Guest").connection("users").uniqueID( user3Username ).post();
+ this.refreshIndex();
+
//8. post an activity to the group
//JSON should look like this:
@@ -448,30 +477,19 @@ public class GroupResourceIT extends AbstractRestIT {
Entity activity = new Entity();
activity.put("actor", payload);
activity.put("verb", "post");
- String content = "content";
+ final String content = "Wilma cannot see this!";
activity.put("content", content);
- Entity activityResponse = this.app().collection("groups").uniqueID(groupName).connection("activities").post(activity);
+ Entity activityResponse = this.app().collection("groups")
+ .uniqueID(groupName).connection("activities").post(activity);
assertEquals(activityResponse.get("content"), content);
this.refreshIndex();
- //9. delete the default role
- this.app().collection("role").uniqueID("Default").delete();
-/*
- //10. add permissions to group: {"permission":"get,post,put,delete:/groups/${group}/**"}'
- payload = new Entity();
- String permission = "get,post,put,delete:/groups/${group}/**";
- payload.put("permission",permission);
- Entity permissionResponse = this.app().collection("groups").entity(group).connection("permissions").post(payload);
- assertEquals(permissionResponse.get("data"), permission);
-*/
//11. log user1 in, should then be using the app user's token not the admin token
this.getAppUserToken(user1Username, password);
-
- //TODO: next failing currently because permissions seem to be borked in the stack
-
- //12. make sure the activity appears in the feed of user 1
- Collection user1ActivityResponse = this.app().collection("groups").entity(group).connection("activities").get();
+ //10. make sure the activity appears in the feed of user 1
+ Collection user1ActivityResponse = this.app().collection("groups")
+ .entity(group).connection("activities").get();
boolean found = false;
while (user1ActivityResponse.hasNext()) {
Entity tempActivity = user1ActivityResponse.next();
@@ -481,11 +499,12 @@ public class GroupResourceIT extends AbstractRestIT {
}
assertTrue(found);
- //13. log user2 in, should then be using the app user's token not the admin token
+ //11. log user2 in, should then be using the app user's token not the admin token
this.getAppUserToken(user2Username, password);
- //14. make sure the activity appears in the feed of user 2
- Collection user2ActivityResponse = this.app().collection("groups").entity(group).connection("activities").get();
+ //12. make sure the activity appears in the feed of user 2
+ Collection user2ActivityResponse = this.app().collection("groups")
+ .entity(group).connection("activities").get();
found = false;
while (user2ActivityResponse.hasNext()) {
Entity tempActivity = user2ActivityResponse.next();
@@ -495,11 +514,35 @@ public class GroupResourceIT extends AbstractRestIT {
}
assertTrue(found);
- //15. log user3 in, should then be using the app user's token not the admin token
+ //13. log user3 in, should then be using the app user's token not the admin token
this.getAppUserToken(user3Username, password);
- //16. make sure the activity does not appear in the feed of user 3, since they are not part of the group
- Collection user3ActivityResponse = this.app().collection("groups").entity(group).connection("activities").get();
+ //14. user3 should be denied access to the group's activities
+ try {
+
+ Collection user3ActivityResponse = this.app().collection( "groups" )
+ .entity( group ).connection( "activities" ).get();
+ fail("user3 should have been denied access to the group's activities");
+
+ } catch (NotAuthorizedException expected) {}
+
+
+ //15. check that activity appears in user 1's personal feed
+ this.getAppUserToken(user1Username, password);
+ user1ActivityResponse = this.app().collection("activities").get();
+ found = false;
+ while (user1ActivityResponse.hasNext()) {
+ Entity tempActivity = user1ActivityResponse.next();
+ if (tempActivity.get("type").equals("activity") && tempActivity.get("content").equals(content)) {
+ found = true;
+ }
+ }
+ assertTrue(found);
+
+
+ //16. check that activity does not appear user 3's personal feed
+ this.getAppUserToken(user3Username, password);
+ Collection user3ActivityResponse = this.app().collection("activities").get();
found = false;
while (user3ActivityResponse.hasNext()) {
Entity tempActivity = user3ActivityResponse.next();
@@ -508,13 +551,10 @@ public class GroupResourceIT extends AbstractRestIT {
}
}
assertFalse(found);
-
-
}
public void updateGroupWithSameNameAsApp() throws IOException {
-
// create groupMap with same name as app
String groupPath = this.clientSetup.getAppName();
String groupName = this.clientSetup.getAppName();
@@ -534,10 +574,5 @@ public class GroupResourceIT extends AbstractRestIT {
groupResponse = this.app().collection("groups").uniqueID(uuid).put(group);
assertEquals(groupResponse.get("title"), evenNewerTitle);
this.refreshIndex();
-
-
-
}
-
-
}
[11/29] usergrid git commit: Fixes issue that caused shards to get
removed prematurely.
Posted by sn...@apache.org.
Fixes issue that caused shards to get removed prematurely.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9bc22410
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9bc22410
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9bc22410
Branch: refs/heads/USERGRID-872
Commit: 9bc22410dd785482f715faec4cbfeae0e712a502
Parents: 784fe51
Author: Michael Russo <mi...@gmail.com>
Authored: Wed Nov 18 15:48:07 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Wed Nov 18 15:48:07 2015 -0800
----------------------------------------------------------------------
.../impl/EdgeSerializationImpl.java | 61 ++++++++++++++++++--
.../shard/impl/EdgeShardSerializationImpl.java | 3 +-
.../shard/impl/ShardGroupColumnIterator.java | 4 +-
.../shard/impl/ShardGroupCompactionImpl.java | 2 +-
.../impl/shard/EdgeShardSerializationTest.java | 44 ++++++++++++++
5 files changed, 106 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java
index 9e25946..0f4d722 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java
@@ -26,15 +26,15 @@ import java.util.UUID;
import javax.inject.Inject;
+import com.google.common.base.Optional;
import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.graph.GraphFig;
-import org.apache.usergrid.persistence.graph.MarkedEdge;
-import org.apache.usergrid.persistence.graph.SearchByEdge;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.SearchByIdType;
+import org.apache.usergrid.persistence.graph.*;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
@@ -303,6 +303,15 @@ public class EdgeSerializationImpl implements EdgeSerialization {
protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) {
return shardedEdgeSerialization.getEdgeVersions( edgeColumnFamilies, scope, search, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange( final Collection<Shard> readShards ) {
+
+ final SearchByEdge searchFullRange = new SimpleSearchByEdge(
+ search.sourceNode(), search.getType(),search.targetNode(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent());
+
+ return shardedEdgeSerialization.getEdgeVersions( edgeColumnFamilies, scope, searchFullRange, readShards );
+ }
};
}
@@ -329,6 +338,15 @@ public class EdgeSerializationImpl implements EdgeSerialization {
protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) {
return shardedEdgeSerialization.getEdgesFromSource( edgeColumnFamilies, scope, edgeType, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange( final Collection<Shard> readShards ) {
+
+ final SearchByEdgeType searchFullRange = new SimpleSearchByEdgeType(
+ edgeType.getNode(), edgeType.getType(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent(), false );
+
+ return shardedEdgeSerialization.getEdgesFromSource( edgeColumnFamilies, scope, searchFullRange, readShards );
+ }
};
}
@@ -357,6 +375,17 @@ public class EdgeSerializationImpl implements EdgeSerialization {
return shardedEdgeSerialization
.getEdgesFromSourceByTargetType( edgeColumnFamilies, scope, edgeType, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange (final Collection<Shard> readShards) {
+
+ final SearchByIdType edgeTypeFullRange = new SimpleSearchByIdType(
+ edgeType.getNode(), edgeType.getType(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ edgeType.getIdType(), Optional.absent(), false );
+
+ return shardedEdgeSerialization
+ .getEdgesFromSourceByTargetType( edgeColumnFamilies, scope, edgeTypeFullRange, readShards);
+ }
};
}
@@ -382,6 +411,17 @@ public class EdgeSerializationImpl implements EdgeSerialization {
protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) {
return shardedEdgeSerialization.getEdgesToTarget( edgeColumnFamilies, scope, edgeType, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange( final Collection<Shard> readShards ) {
+
+ final SearchByEdgeType edgeTypeFullRange = new SimpleSearchByEdgeType(
+ edgeType.getNode(), edgeType.getType(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ Optional.absent(), false );
+
+ return shardedEdgeSerialization.getEdgesToTarget( edgeColumnFamilies, scope, edgeTypeFullRange, readShards );
+ }
+
};
}
@@ -411,6 +451,17 @@ public class EdgeSerializationImpl implements EdgeSerialization {
return shardedEdgeSerialization
.getEdgesToTargetBySourceType( edgeColumnFamilies, scope, edgeType, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange( final Collection<Shard> readShards ) {
+
+ final SearchByIdType edgeTypeFullRange = new SimpleSearchByIdType(
+ edgeType.getNode(), edgeType.getType(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ edgeType.getIdType(), Optional.absent(), false );
+
+ return shardedEdgeSerialization.getEdgesToTargetBySourceType( edgeColumnFamilies, scope, edgeTypeFullRange, readShards);
+
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
index f107307..2f83d6f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
@@ -156,7 +156,8 @@ public class EdgeShardSerializationImpl implements EdgeShardSerialization {
final MutationBatch batch = keyspace.prepareMutationBatch();
- batch.withRow( EDGE_SHARDS, rowKey ).deleteColumn( shard.getShardIndex() );
+ batch.withTimestamp(shard.getCreatedTime()).withRow( EDGE_SHARDS, rowKey )
+ .deleteColumn( shard.getShardIndex() );
return batch;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java
index 72b617f..9604e63 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java
@@ -117,6 +117,8 @@ public abstract class ShardGroupColumnIterator implements Iterator<MarkedEdge> {
*/
protected abstract Iterator<MarkedEdge> getIterator( Collection<Shard> readShards );
+ protected abstract Iterator<MarkedEdge> getIteratorFullRange( Collection<Shard> readShards );
+
public boolean advance() {
@@ -141,7 +143,7 @@ public abstract class ShardGroupColumnIterator implements Iterator<MarkedEdge> {
logger.trace( "Our shard is empty, we need to perform an audit on shard group {}", group );
//fire and forget if we miss it here, we'll get it next read
- shardGroupDeletion.maybeDeleteShard(this.applicationScope, this.directedEdgeMeta, group, getIterator( group.getReadShards() ) );
+ shardGroupDeletion.maybeDeleteShard(this.applicationScope, this.directedEdgeMeta, group, getIteratorFullRange( group.getReadShards() ) );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index 7dd0521..e663d5a 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -150,7 +150,7 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
.checkArgument( group.shouldCompact( startTime ), "Compaction cannot be run yet. Ignoring compaction." );
if(LOG.isDebugEnabled()) {
- LOG.debug("Compacting shard group. count is {} ", countAudits.get());
+ LOG.debug("Compacting shard group. Audit count is {} ", countAudits.get());
}
final CompactionResult.CompactionBuilder resultBuilder = CompactionResult.builder();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
index 2641ed7..1f8bfa9 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
@@ -40,6 +40,8 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
@@ -52,6 +54,8 @@ import static org.mockito.Mockito.when;
@UseModules({ TestGraphModule.class })
public class EdgeShardSerializationTest {
+ private static final Logger log = LoggerFactory.getLogger( EdgeShardSerializationTest.class );
+
@Inject
@Rule
@@ -203,4 +207,44 @@ public class EdgeShardSerializationTest {
assertFalse( results.hasNext() );
}
+
+ @Test
+ public void sameShardIndexRemoval() throws ConnectionException {
+
+ final Id now = IdGenerator.createId( "test" );
+
+ final long timestamp = 2000L;
+
+ final Shard shard1 = new Shard( 1000L, timestamp, false );
+ final Shard shard2 = new Shard( shard1.getShardIndex(), timestamp * 2, true );
+
+
+ final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType( now, "edgeType", "subType" );
+ MutationBatch batch = edgeShardSerialization.writeShardMeta( scope, shard1, sourceEdgeMeta );
+ batch.mergeShallow( edgeShardSerialization.writeShardMeta( scope, shard2, sourceEdgeMeta ) );
+ batch.execute();
+
+
+ Iterator<Shard> results =
+ edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
+
+ // Latest timestamp comes first
+ assertEquals( shard2, results.next() );
+
+ // This should now not remove anything
+ edgeShardSerialization.removeShardMeta( scope, shard1, sourceEdgeMeta ).execute();
+
+
+ // Get iterator again
+ results = edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
+
+ // We should still have shard3 stored
+ assertEquals( shard2, results.next() );
+
+
+
+ }
+
+
+
}
[14/29] usergrid git commit: Update filter to allow removal of all
entities older than the last marked version.
Posted by sn...@apache.org.
Update filter to allow removal of all entities older than the last marked version.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/faa234a8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/faa234a8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/faa234a8
Branch: refs/heads/USERGRID-872
Commit: faa234a85f1526a17a6f7d3049a7cb0820816190
Parents: 88f6fea
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 10:51:22 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 10:51:22 2015 -0800
----------------------------------------------------------------------
.../usergrid/corepersistence/asyncevents/EventBuilderImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faa234a8/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 9b68c4c..f7f7ba3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -144,7 +144,7 @@ public class EventBuilderImpl implements EventBuilder {
ecmDeleteObservable =
ecm.getVersions( entityId )
- .filter( mvccLogEntry-> mvccLogEntry.getState() == MvccLogEntry.State.DELETED)
+ .filter( mvccLogEntry-> mvccLogEntry.getVersion().timestamp() <= mostRecentlyMarked.getVersion().timestamp())
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
[17/29] usergrid git commit: Fix it to be <=
Posted by sn...@apache.org.
Fix it to be <=
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/be9bcb0b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/be9bcb0b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/be9bcb0b
Branch: refs/heads/USERGRID-872
Commit: be9bcb0b3f84f2ba09db22f7ea25c42762a44ddd
Parents: 1b1ed8b
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 12:23:44 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 12:23:44 2015 -0800
----------------------------------------------------------------------
.../usergrid/corepersistence/asyncevents/EventBuilderImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be9bcb0b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index c9ae6da..48df8d0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -145,7 +145,7 @@ public class EventBuilderImpl implements EventBuilder {
ecmDeleteObservable =
ecm.getVersions( entityId )
- .filter( mvccLogEntry-> UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) < 0)
+ .filter( mvccLogEntry-> UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) <= 0)
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
[03/29] usergrid git commit: Added additional test for isNew in
shardEntryGroup
Posted by sn...@apache.org.
Added additional test for isNew in shardEntryGroup
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3a6fd944
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3a6fd944
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3a6fd944
Branch: refs/heads/USERGRID-872
Commit: 3a6fd9444f7687f495f2d13fea2b6bb8ccc5d0e5
Parents: 2b2793f
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 16 19:08:04 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 16 19:08:04 2015 -0700
----------------------------------------------------------------------
.../impl/shard/ShardEntryGroupTest.java | 31 ++++++++++++++++++++
1 file changed, 31 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a6fd944/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
index 28d42d2..2f0dfaa 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
@@ -434,6 +434,37 @@ public class ShardEntryGroupTest {
}
+
+
+ @Test
+ public void testIsNew() {
+ //set with no shard
+
+ final long delta = 10000;
+
+ //created at 10000
+ final Shard firstShard = new Shard( 10000, 10000, true );
+
+ final Shard secondShard = new Shard( 10000, 10001, true );
+
+ final ShardEntryGroup shardGroup = new ShardEntryGroup( delta );
+
+ shardGroup.addShard( secondShard );
+ shardGroup.addShard( firstShard );
+
+ final boolean resultCreateTime = shardGroup.isNew( secondShard.getCreatedTime() );
+
+ assertTrue( "This is a new shard", resultCreateTime );
+
+ final boolean resultEqualToDelta = shardGroup.isNew( secondShard.getCreatedTime() + delta );
+
+ assertTrue( "This is a new shard", resultEqualToDelta );
+
+
+ final boolean greaterThan = shardGroup.isNew( secondShard.getCreatedTime() + delta + 1 );
+
+ assertFalse( "This is not a new shard", greaterThan );
+ }
}
[13/29] usergrid git commit: Fix delete so that it doesn't return null
Posted by sn...@apache.org.
Fix delete so that it doesn't return null
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/88f6feae
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/88f6feae
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/88f6feae
Branch: refs/heads/USERGRID-872
Commit: 88f6feae8c8d635598954ee9a5d239eda42653bd
Parents: 9bc2241
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 09:43:36 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 09:43:36 2015 -0800
----------------------------------------------------------------------
.../asyncevents/EventBuilderImpl.java | 39 +++++++++-----------
1 file changed, 17 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/88f6feae/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index d819f39..9b68c4c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -125,42 +125,37 @@ public class EventBuilderImpl implements EventBuilder {
log.debug( "Deleting entity id from index in app scope {} with entityId {} }", applicationScope, entityId );
final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
-
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ //TODO USERGRID-1123: Implement so we don't iterate logs twice (latest DELETED version, then to get all DELETED)
- //TODO: change this to be an observable
- //so we get these versions and loop through them until we find the MvccLogEntry that is marked as delete.
- //TODO: evauluate this to possibly be an observable to pass to the nextmethod.
MvccLogEntry mostRecentlyMarked = ecm.getVersions( entityId ).toBlocking()
.firstOrDefault( null,
mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
- //If there is nothing marked then we shouldn't return any results.
- //TODO: evaluate if we want to return null or return empty observable when we don't have any results marked as deleted.
- if(mostRecentlyMarked == null)
- return null;
+ // De-indexing and entity deletes don't check log entiries. We must do that first. If no DELETED logs, then
+ // return an empty observable as our no-op.
+ Observable<IndexOperationMessage> deIndexObservable = Observable.empty();
+ Observable<List<MvccLogEntry>> ecmDeleteObservable = Observable.empty();
- //observable of index operation messages
- //this method will need the most recent version.
- //When we go to compact the graph make sure you turn on the debugging mode for the deleted nodes so
- //we can verify that we mark them. That said that part seems kinda done. as we also delete the mvcc buffers.
- final Observable<IndexOperationMessage> edgeObservable =
- indexService.deleteEntityIndexes( applicationScope, entityId, mostRecentlyMarked.getVersion() );
+ if(mostRecentlyMarked != null){
+ deIndexObservable =
+ indexService.deleteEntityIndexes( applicationScope, entityId, mostRecentlyMarked.getVersion() );
+ ecmDeleteObservable =
+ ecm.getVersions( entityId )
+ .filter( mvccLogEntry-> mvccLogEntry.getState() == MvccLogEntry.State.DELETED)
+ .buffer( serializationFig.getBufferSize() )
+ .doOnNext( buffer -> ecm.delete( buffer ) );
- //TODO: not sure what we need the list of versions here when we search for the mark above
- //observable of entries as the batches are deleted
- final Observable<List<MvccLogEntry>> entries =
- ecm.getVersions( entityId ).buffer( serializationFig.getBufferSize() )
- .doOnNext( buffer -> ecm.delete( buffer ) );
+ }
- // observable of the edge delete from graph
- final Observable<Id> compactedNode = gm.compactNode(entityId);
+ // Graph compaction checks the versions inside compactNode, just build this up for the caller to subscribe to
+ final Observable<Id> graphCompactObservable = gm.compactNode(entityId);
- return new EntityDeleteResults( edgeObservable, entries, compactedNode );
+ return new EntityDeleteResults( deIndexObservable, ecmDeleteObservable, graphCompactObservable );
}
[09/29] usergrid git commit: Fixes the 2.5 checks in the test
Posted by sn...@apache.org.
Fixes the 2.5 checks in the test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8ba0ef01
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8ba0ef01
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8ba0ef01
Branch: refs/heads/USERGRID-872
Commit: 8ba0ef01a90ac1352ea56203f07c042d73e24306
Parents: a33f616
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 11:19:06 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 11:19:06 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/NodeShardAllocationTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8ba0ef01/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
index bc364cc..6671dec 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
@@ -84,7 +84,7 @@ public class NodeShardAllocationTest {
final long timeout = 30000;
when( graphFig.getShardCacheTimeout() ).thenReturn( timeout );
- when( graphFig.getShardMinDelta() ).thenReturn( timeout * 2 );
+ when( graphFig.getShardMinDelta() ).thenReturn( ( long ) (timeout * 2.5) );
}
@@ -111,7 +111,7 @@ public class NodeShardAllocationTest {
when( timeService.getCurrentTime() ).thenReturn( timeservicetime );
- final long expected = timeservicetime - 2 * graphFig.getShardCacheTimeout();
+ final long expected = ( long ) (timeservicetime - 2.5 * graphFig.getShardCacheTimeout());
final long returned = approximation.getMinTime();
@@ -722,7 +722,7 @@ public class NodeShardAllocationTest {
//now test something that passes.
- final long minDelta = cacheTimeout * 2;
+ final long minDelta = ( long ) (cacheTimeout * 2.5);
when( graphFig.getShardMinDelta() ).thenReturn( minDelta );
[08/29] usergrid git commit: Adds more logging
Posted by sn...@apache.org.
Adds more logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a33f616b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a33f616b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a33f616b
Branch: refs/heads/USERGRID-872
Commit: a33f616beed86f475999a5904520776494891968
Parents: 06c9c10
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 11:17:07 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 11:17:07 2015 -0700
----------------------------------------------------------------------
.../impl/shard/impl/NodeShardAllocationImpl.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a33f616b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index 0cc12b6..d52f807 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -138,11 +138,13 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
* Nothing to do, it's been created very recently, we don't create a new one
*/
if ( shardEntryGroup.isCompactionPending() ) {
+ LOG.trace( "Shard entry group {} is compacting, not auditing", shardEntryGroup );
return false;
}
//we can't allocate, we have more than 1 write shard currently. We need to compact first
if ( shardEntryGroup.entrySize() != 1 ) {
+ LOG.trace( "Shard entry group {} does not have 1 entry, not allocating", shardEntryGroup );
return false;
}
@@ -151,9 +153,12 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
* Check the min shard in our system
*/
final Shard shard = shardEntryGroup.getMinShard();
+ final long minTime = getMinTime();
- if ( shard.getCreatedTime() >= getMinTime() ) {
+
+ if ( shard.getCreatedTime() >= minTime ) {
+ LOG.trace( "Shard entry group {} and shard {} is before the minimum created time of {}. Not allocating.does not have 1 entry, not allocating", shardEntryGroup, shard, minTime );
return false;
}
@@ -253,7 +258,7 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
@Override
public long getMinTime() {
- final long minimumAllowed = 2 * graphFig.getShardCacheTimeout();
+ final long minimumAllowed = ( long ) (2.5 * graphFig.getShardCacheTimeout());
final long minDelta = graphFig.getShardMinDelta();
[25/29] usergrid git commit: PR changes test code only. This closes
#443
Posted by sn...@apache.org.
PR changes test code only. This closes #443
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/41521881
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/41521881
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/41521881
Branch: refs/heads/USERGRID-872
Commit: 41521881e665835c53167d5943e99ddd9289957c
Parents: 59b1f0d c1f5e16
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Nov 23 15:10:46 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Nov 23 15:10:46 2015 -0500
----------------------------------------------------------------------
.../applications/ApplicationResourceIT.java | 33 ++++++++++++--------
1 file changed, 20 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
[29/29] usergrid git commit: Update tools to use 2.1.1 and add
missing ES args to two export tools.
Posted by sn...@apache.org.
Update tools to use 2.1.1 and add missing ES args to two export tools.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bfeb91b4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bfeb91b4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bfeb91b4
Branch: refs/heads/USERGRID-872
Commit: bfeb91b48c4e33649937372c307465dcd1c407fb
Parents: 668a801
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Dec 1 09:25:18 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Dec 1 09:25:18 2015 -0500
----------------------------------------------------------------------
stack/tools/pom.xml | 2 +-
.../test/java/org/apache/usergrid/tools/ExportAppTest.java | 9 ++++++---
.../org/apache/usergrid/tools/ExportImportAdminsTest.java | 6 ++++--
3 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bfeb91b4/stack/tools/pom.xml
----------------------------------------------------------------------
diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml
index 0e20e91..6b65268 100644
--- a/stack/tools/pom.xml
+++ b/stack/tools/pom.xml
@@ -29,7 +29,7 @@
<name>Usergrid Tools</name>
<description>Command line tools for Usergrid system.</description>
<packaging>jar</packaging>
- <version>2.1.0-SNAPSHOT</version>
+ <version>2.1.1-SNAPSHOT</version>
<reporting>
<plugins>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bfeb91b4/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
index 890c972..7b7272e 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
@@ -29,6 +29,7 @@ import java.io.FileFilter;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.apache.usergrid.StressTest;
/**
* TODO: better test, this is really just a smoke test.
@@ -55,9 +56,11 @@ public class ExportAppTest {
ExportDataCreator creator = new ExportDataCreator();
creator.startTool( new String[] {
- "-organization", orgName,
- "-application", appName,
- "-host", "localhost:9120"
+ "-organization", orgName,
+ "-application", appName,
+ "-host", "localhost:9120",
+ "-eshost", "localhost:9200",
+ "-escluster", "usergrid"
}, false);
long start = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bfeb91b4/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
index 23ff096..5573279 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
@@ -216,8 +216,10 @@ public class ExportImportAdminsTest {
ImportAdmins importAdmins = new ImportAdmins();
importAdmins.startTool( new String[]{
- "-host", "localhost:9120",
- "-inputDir", tempDir.getAbsolutePath()
+ "-host", "localhost:9120",
+ "-eshost", "localhost:9200",
+ "-escuster", "usergrid",
+ "-inputDir", tempDir.getAbsolutePath()
}, false );
// verify that users and orgs were created correctly
[06/29] usergrid git commit: Fixes logging statement
Posted by sn...@apache.org.
Fixes logging statement
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4f00a437
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4f00a437
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4f00a437
Branch: refs/heads/USERGRID-872
Commit: 4f00a437e556d8c352107a58767fb19590b62875
Parents: c44dfda
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 10:06:54 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 10:06:54 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/impl/NodeShardAllocationImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4f00a437/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index 374d373..0cc12b6 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -226,7 +226,7 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
* Sanity check in case we audit before we have a full shard
*/
if ( marked == null ) {
- LOG.trace( "Shard {} in shard group {} not full, not splitting", shardEntryGroup );
+ LOG.trace( "Shard {} in shard group {} not full, not splitting", shard, shardEntryGroup );
return false;
}
[24/29] usergrid git commit: PR changes test code only. This closes
#441
Posted by sn...@apache.org.
PR changes test code only. This closes #441
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/59b1f0d4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/59b1f0d4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/59b1f0d4
Branch: refs/heads/USERGRID-872
Commit: 59b1f0d44dd2f630120547b3d851faf3c213c5f0
Parents: 6fffe93 28a627a
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Nov 23 15:09:52 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Nov 23 15:09:52 2015 -0500
----------------------------------------------------------------------
.../collection/groups/GroupResourceIT.java | 115 ++++++++++++-------
1 file changed, 75 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
[18/29] usergrid git commit: format change only
Posted by sn...@apache.org.
format change only
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e5b97304
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e5b97304
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e5b97304
Branch: refs/heads/USERGRID-872
Commit: e5b97304ebe2dcccce32ea18c7d7994130ca5738
Parents: be9bcb0
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Nov 19 15:29:39 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Nov 19 15:29:39 2015 -0500
----------------------------------------------------------------------
.../corepersistence/asyncevents/EventBuilderImpl.java | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5b97304/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 48df8d0..d624132 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -131,8 +131,7 @@ public class EventBuilderImpl implements EventBuilder {
//TODO USERGRID-1123: Implement so we don't iterate logs twice (latest DELETED version, then to get all DELETED)
MvccLogEntry mostRecentlyMarked = ecm.getVersions( entityId ).toBlocking()
- .firstOrDefault( null,
- mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
+ .firstOrDefault( null, mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
// De-indexing and entity deletes don't check log entiries. We must do that first. If no DELETED logs, then
// return an empty observable as our no-op.
@@ -145,14 +144,12 @@ public class EventBuilderImpl implements EventBuilder {
ecmDeleteObservable =
ecm.getVersions( entityId )
- .filter( mvccLogEntry-> UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) <= 0)
+ .filter( mvccLogEntry->
+ UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) <= 0)
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
-
-
}
-
// Graph compaction checks the versions inside compactNode, just build this up for the caller to subscribe to
final Observable<Id> graphCompactObservable = gm.compactNode(entityId);
[07/29] usergrid git commit: Fixes log statement
Posted by sn...@apache.org.
Fixes log statement
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/06c9c101
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/06c9c101
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/06c9c101
Branch: refs/heads/USERGRID-872
Commit: 06c9c101242c125e4832f89bf6e6884174cb782e
Parents: 4f00a43
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 10:54:52 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 10:54:52 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/impl/ShardGroupCompactionImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06c9c101/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index 21f2d72..7dd0521 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -307,8 +307,9 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
countAudits.getAndIncrement();
if(LOG.isDebugEnabled()) {
- LOG.debug("Auditing shard group. count is {} ", countAudits.get());
+ LOG.debug("Auditing shard group {}. count is {} ", group, countAudits.get());
}
+
/**
* Try and submit. During back pressure, we may not be able to submit, that's ok. Better to drop than to
* hose the system
[04/29] usergrid git commit: Added additional logging statement
Posted by sn...@apache.org.
Added additional logging statement
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ad12f359
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ad12f359
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ad12f359
Branch: refs/heads/USERGRID-872
Commit: ad12f3599d2d66e799f6c3294f6e33d8e1c5b4d4
Parents: 3a6fd94
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 08:30:50 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 08:30:50 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/impl/ShardGroupDeletionImpl.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ad12f359/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index d289f30..c460a5d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -178,6 +178,9 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
}
+ logger.info( "Deleting shard {} at time {} ", shard, timeService.getCurrentTime() );
+
+
final MutationBatch shardRemovalMutation =
edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
[26/29] usergrid git commit: PR changes test code only. This closes
#444
Posted by sn...@apache.org.
PR changes test code only. This closes #444
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5dfdf81e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5dfdf81e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5dfdf81e
Branch: refs/heads/USERGRID-872
Commit: 5dfdf81e8d0c58e04b305b3bd1adeac176fbaae5
Parents: 4152188 46f9802
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Nov 23 15:11:05 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Nov 23 15:11:05 2015 -0500
----------------------------------------------------------------------
.../usergrid/rest/management/AdminUsersIT.java | 253 +++++++++++--------
1 file changed, 150 insertions(+), 103 deletions(-)
----------------------------------------------------------------------