You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/09/02 00:51:20 UTC

[01/35] usergrid git commit: fix email regex

Repository: usergrid
Updated Branches:
  refs/heads/master ae39eca6a -> aa0353d96


fix email regex


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ca49f1fd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ca49f1fd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ca49f1fd

Branch: refs/heads/master
Commit: ca49f1fde1145905bfa2d15b746bf2a80b2b7e30
Parents: b5ed32a
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 23 09:40:22 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 23 09:40:22 2016 -0700

----------------------------------------------------------------------
 .../org/apache/usergrid/persistence/index/query/Identifier.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/ca49f1fd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
index 9313c03..f98ccc7 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
@@ -33,7 +33,7 @@ public class Identifier implements Serializable {
 
     public static final String UUID_REX = 
             "[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}";
-    public static final String EMAIL_REX =  "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,4}";
+    public static final String EMAIL_REX =  "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}";
 
     public enum Type {
         UUID, NAME, EMAIL


[31/35] usergrid git commit: Use StringUtils.isEmpty() to determine if region is not set.

Posted by mr...@apache.org.
Use StringUtils.isEmpty() to determine if region is not set.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9bb0ae6d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9bb0ae6d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9bb0ae6d

Branch: refs/heads/master
Commit: 9bb0ae6d099968079675cecb92e034b978f89006
Parents: 69cfbbd
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Sep 1 12:59:39 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Sep 1 12:59:39 2016 -0400

----------------------------------------------------------------------
 .../persistence/collection/mvcc/stage/write/WriteCommit.java    | 5 +++--
 .../collection/mvcc/stage/write/WriteUniqueVerify.java          | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bb0ae6d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index 5b98ca5..553ac4c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
 
+import org.apache.commons.lang3.StringUtils;
 import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
 import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
 import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException;
@@ -136,10 +137,10 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Collect
         // akkaFig may be null when this is called from JUnit tests
         if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
             String region = ioEvent.getRegion();
-            if ( region == null ) {
+            if ( StringUtils.isEmpty(region) ) {
                 region = uniqueValuesFig.getAuthoritativeRegion();
             }
-            if ( region == null ) {
+            if ( StringUtils.isEmpty(region) ) {
                 region = actorSystemFig.getRegionLocal();
             }
             confirmUniqueFieldsAkka( mvccEntity, version, applicationScope, region );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bb0ae6d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index a3565ea..7a63061 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -29,6 +29,7 @@ import com.netflix.hystrix.HystrixCommand;
 import com.netflix.hystrix.HystrixCommandGroupKey;
 import com.netflix.hystrix.HystrixCommandProperties;
 import com.netflix.hystrix.HystrixThreadPoolProperties;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
 import org.apache.usergrid.persistence.collection.MvccEntity;
 import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
@@ -124,10 +125,10 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
         final ApplicationScope applicationScope = ioevent.getEntityCollection();
 
         String region = ioevent.getRegion();
-        if ( region == null ) {
+        if ( StringUtils.isEmpty(region) ) {
             region = uniqueValuesFig.getAuthoritativeRegion();
         }
-        if ( region == null ) {
+        if ( StringUtils.isEmpty(region) ) {
             region = actorSystemFig.getRegionLocal();
         }
         try {


[27/35] usergrid git commit: fix duplicate user test to use random usernames in case DB cleanup not done correctly

Posted by mr...@apache.org.
fix duplicate user test to use random usernames in case DB cleanup not
done correctly


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9520496b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9520496b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9520496b

Branch: refs/heads/master
Commit: 9520496b819ad2e692fd9da2e56b7a46c54079f3
Parents: f73ce4a
Author: Mike Dunker <md...@apigee.com>
Authored: Wed Aug 31 12:29:55 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Wed Aug 31 12:29:55 2016 -0700

----------------------------------------------------------------------
 .../collection/uniquevalues/UniqueValuesServiceTest.java         | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9520496b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
index 1788a88..520b9af 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
@@ -47,6 +47,7 @@ import org.slf4j.LoggerFactory;
 import rx.Observable;
 
 import java.util.Collection;
+import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -137,9 +138,10 @@ public class UniqueValuesServiceTest extends AbstractUniqueValueTest {
 
         for (int i = 0; i < numUsers; i++) {
 
+            String username = "user_" + UUID.randomUUID().toString();
+
             // multiple threads simultaneously trying to create a user with the same propertyName
             for (int j = 0; j < numThreads; j++) {
-                String username = "user_" + i;
 
                 execService.submit( () -> {
 


[13/35] usergrid git commit: Add ShardManager tool.

Posted by mr...@apache.org.
Add ShardManager tool.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/34b1697c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/34b1697c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/34b1697c

Branch: refs/heads/master
Commit: 34b1697c507cc2d67cb2f9c6903b59e2080327c2
Parents: bf34a28
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Aug 28 11:07:31 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Aug 28 11:07:31 2016 -0700

----------------------------------------------------------------------
 .../org/apache/usergrid/tools/ShardManager.java | 214 +++++++++++++++++++
 1 file changed, 214 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/34b1697c/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java
new file mode 100644
index 0000000..ecf447c
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.util.*;
+
+import com.google.common.base.Optional;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardSerialization;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
+import org.apache.usergrid.persistence.model.entity.*;
+
+import org.apache.usergrid.utils.InflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import sun.security.provider.SHA;
+
+import static org.apache.commons.lang.StringUtils.isBlank;
+import static org.apache.commons.lang.StringUtils.isEmpty;
+import static org.apache.commons.lang.StringUtils.isNotEmpty;
+
+
+public class ShardManager extends ToolBase {
+
+    private static final Logger logger = LoggerFactory.getLogger(ShardManager.class);
+
+    private static final String APPLICATION_ARG = "app";
+
+    private static final String ENTITY_TYPE_ARG = "entityType";
+
+    private static final String REPAIR_TASK = "repairTask";
+
+    private static final String SHARD_TYPE_ARG = "shardType";
+
+
+    @Override
+    @SuppressWarnings("static-access")
+    public Options createOptions() {
+
+
+        Options options = super.createOptions();
+
+
+        Option appOption = OptionBuilder.withArgName(APPLICATION_ARG).hasArg().isRequired(true)
+            .withDescription("application id").create(APPLICATION_ARG);
+
+        options.addOption(appOption);
+
+        Option collectionOption =
+            OptionBuilder.withArgName(ENTITY_TYPE_ARG).hasArg().isRequired(true).withDescription("singular collection name")
+                .create(ENTITY_TYPE_ARG);
+
+        options.addOption(collectionOption);
+
+        Option repairOption =
+            OptionBuilder.withArgName(REPAIR_TASK).hasArg().isRequired(false).withDescription("repair task to execute")
+                .create(REPAIR_TASK);
+
+        options.addOption(repairOption);
+
+        Option shardTypeOption =
+            OptionBuilder.withArgName(SHARD_TYPE_ARG).hasArg().isRequired(true).withDescription("either collection or connection")
+                .create(SHARD_TYPE_ARG);
+
+        options.addOption(shardTypeOption);
+
+        return options;
+    }
+
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see
+     * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+     */
+    @Override
+    public void runTool(CommandLine line) throws Exception {
+
+        startSpring();
+
+        if (line.getOptionValue(APPLICATION_ARG).isEmpty()) {
+            throw new RuntimeException("Application ID not provided.");
+        }
+        final UUID app = UUID.fromString(line.getOptionValue(APPLICATION_ARG));
+
+        String entityType = line.getOptionValue(ENTITY_TYPE_ARG);
+
+        String repairTask = line.getOptionValue(REPAIR_TASK);
+
+        String shardType = line.getOptionValue(SHARD_TYPE_ARG);
+
+        boolean repair = false;
+        if( isNotEmpty(repairTask) && (
+            repairTask.equalsIgnoreCase("removeAllShardEnds") || repairTask.equalsIgnoreCase("removeLastShardEnd") ||
+            repairTask.equalsIgnoreCase("resetAllCompactionStatus"))) {
+
+            repair = true;
+        }
+
+
+        logger.info("Starting Tool: ShardManager");
+        logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));
+
+        EntityRef headEntity = new SimpleEntityRef("application", app);
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(app, "application"));
+        EdgeShardSerialization edgeShardSerialization = injector.getInstance(EdgeShardSerialization.class);
+
+        String collectionName = InflectionUtils.pluralize(entityType);
+
+        // default to assume collection
+        String metaType = CpNamingUtils.getEdgeTypeFromCollectionName(collectionName);
+
+            if( isNotEmpty(shardType) ){
+
+            if( shardType.equalsIgnoreCase("collection")){
+                metaType = CpNamingUtils.getEdgeTypeFromCollectionName(collectionName);
+
+            }else if( shardType.equalsIgnoreCase("connection")){
+                metaType = CpNamingUtils.getEdgeTypeFromConnectionType(entityType);
+            }
+        }
+
+
+        final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode(headEntity.asId(), metaType);
+
+        Iterator<Shard> shards = edgeShardSerialization.getShardMetaData(applicationScope, Optional.absent(), directedEdgeMeta);
+
+        boolean firstShard = true;
+        while (shards.hasNext()) {
+            Shard shard = shards.next();
+
+            logger.info("Seeking over shard: {}", shard);
+
+            if(repair) {
+
+                logger.info("Repair enabled with task: {}", repairTask);
+
+                if( repairTask.equalsIgnoreCase("removeLastShardEnd") && firstShard){
+
+                    logger.info("Removing shard end from shard: {}", shard);
+
+                    shard.setShardEnd(Optional.absent());
+                    edgeShardSerialization.writeShardMeta(applicationScope, shard, directedEdgeMeta).execute();
+
+                }else if ( repairTask.equalsIgnoreCase("removeAllShardEnds")){
+
+                    logger.info("Removing shard end from shard: {}", shard);
+
+                    shard.setShardEnd(Optional.absent());
+                    edgeShardSerialization.writeShardMeta(applicationScope, shard, directedEdgeMeta).execute();
+
+                } else if ( repairTask.equalsIgnoreCase("resetAllCompactionStatus")){
+
+                    logger.info("Setting compacted=false for shard: {}", shard);
+
+                    shard.setCompacted(false);
+                    edgeShardSerialization.writeShardMeta(applicationScope, shard, directedEdgeMeta).execute();
+
+                }
+
+                firstShard = false;
+
+
+            }
+
+        }
+
+
+        if(repair) {
+            // do a final walk-through so changes can be verified
+            Iterator<Shard> finalshards = edgeShardSerialization.getShardMetaData(applicationScope, Optional.absent(), directedEdgeMeta);
+
+            while (finalshards.hasNext()) {
+
+                Shard shard = finalshards.next();
+
+                logger.info("Shard after repair: {}", shard);
+
+            }
+        }
+
+
+        logger.info("ShardManager run complete");
+
+
+    }
+}


[23/35] usergrid git commit: Handle empty collections in CollectionIterator.

Posted by mr...@apache.org.
Handle empty collections in CollectionIterator.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2f21b311
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2f21b311
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2f21b311

Branch: refs/heads/master
Commit: 2f21b311d2a60a7f8a281730229669053c8b1e7a
Parents: 7408f3d
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Aug 30 16:17:04 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Aug 30 16:17:04 2016 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/usergrid/tools/CollectionIterator.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/2f21b311/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
index bec844b..26b5f5f 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -267,7 +267,7 @@ public class CollectionIterator extends ToolBase {
 
 
            return markedEdge;
-        }).toBlocking().last();
+        }).toBlocking().lastOrDefault(null);
 
     }
 }


[08/35] usergrid git commit: add repair to collection iterator tool

Posted by mr...@apache.org.
add repair to collection iterator tool


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fd7a75ee
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fd7a75ee
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fd7a75ee

Branch: refs/heads/master
Commit: fd7a75ee0db360293df628f99bbc304e966e93b4
Parents: a7639d5
Author: Mike Dunker <md...@apigee.com>
Authored: Thu Aug 25 21:34:11 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Thu Aug 25 21:34:11 2016 -0700

----------------------------------------------------------------------
 .../usergrid/tools/CollectionIterator.java      | 44 ++++++++++++++------
 1 file changed, 32 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/fd7a75ee/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
index 02188cb..855c363 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -70,6 +70,8 @@ public class CollectionIterator extends ToolBase {
 
     private static final String ENTITY_TYPE_ARG = "entityType";
 
+    private static final String REMOVE_CONNECTIONS_ARG = "removeConnections";
+
     private EntityManager em;
 
     @Override
@@ -87,11 +89,17 @@ public class CollectionIterator extends ToolBase {
         options.addOption( appOption );
 
         Option collectionOption =
-            OptionBuilder.withArgName(ENTITY_TYPE_ARG).hasArg().isRequired( true ).withDescription( "collection name" )
-                .create(ENTITY_TYPE_ARG);
+                OptionBuilder.withArgName(ENTITY_TYPE_ARG).hasArg().isRequired( true ).withDescription( "singular collection name" )
+                        .create(ENTITY_TYPE_ARG);
 
         options.addOption( collectionOption );
 
+        Option removeConnectionsOption =
+            OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg().isRequired( false ).withDescription( "remove orphaned connections" )
+                .create(REMOVE_CONNECTIONS_ARG);
+
+        options.addOption( removeConnectionsOption );
+
 
         return options;
     }
@@ -108,18 +116,22 @@ public class CollectionIterator extends ToolBase {
 
         startSpring();
 
-        UUID appToFilter = null;
-        if (!line.getOptionValue(APPLICATION_ARG).isEmpty()) {
-            appToFilter = UUID.fromString(line.getOptionValue(APPLICATION_ARG));
+        if (line.getOptionValue(APPLICATION_ARG).isEmpty()) {
+            throw new RuntimeException("Application ID not provided.");
         }
+        final UUID app = UUID.fromString(line.getOptionValue(APPLICATION_ARG));
+
+        String removeOrphansOption = line.getOptionValue(REMOVE_CONNECTIONS_ARG);
+        final boolean removeOrphans = !removeOrphansOption.isEmpty() && removeOrphansOption.toLowerCase().equals("yes");
 
         String entityType = line.getOptionValue(ENTITY_TYPE_ARG);
 
-        logger.info("Staring Tool: CollectionIterator");
+        logger.info("Starting Tool: CollectionIterator");
+        logger.info("Orphans {} be deleted", removeOrphans ? "WILL" : "will not");
         logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));
 
-        em = emf.getEntityManager( appToFilter );
-        EntityRef headEntity = new SimpleEntityRef("application", appToFilter);
+        em = emf.getEntityManager( app );
+        EntityRef headEntity = new SimpleEntityRef("application", app);
 
         CollectionService collectionService = injector.getInstance(CollectionService.class);
         String collectionName = InflectionUtils.pluralize(entityType);
@@ -132,8 +144,6 @@ public class CollectionIterator extends ToolBase {
 
         CollectionInfo collection = getDefaultSchema().getCollection(headEntity.getType(), collectionName);
 
-        final UUID app = appToFilter;
-
 
         IdQueryExecutor idQueryExecutor = new IdQueryExecutor(query.getCursor()) {
             @Override
@@ -157,7 +167,8 @@ public class CollectionIterator extends ToolBase {
             ids.forEach( uuid -> {
 
                 try {
-                    org.apache.usergrid.persistence.Entity retrieved = em.get(new SimpleEntityRef(entityType, uuid));
+                    EntityRef entityRef = new SimpleEntityRef(entityType, uuid);
+                    org.apache.usergrid.persistence.Entity retrieved = em.get(entityRef);
 
                     long timestamp = 0;
                     String dateString = "NOT TIME-BASED";
@@ -173,7 +184,16 @@ public class CollectionIterator extends ToolBase {
 
                         logger.info("{} - {} - entity data found", uuid, dateString);
                     }else{
-                        logger.info("{} - {} - entity data NOT found", uuid, dateString);
+                        if (removeOrphans) {
+                            logger.info("{} - {} - entity data NOT found, REMOVING", uuid, dateString);
+                            try {
+                                em.removeFromCollection(headEntity, collectionName, entityRef );
+                            } catch (Exception e) {
+                                logger.error("{} - exception while trying to remove orphaned connection, {}", uuid, e.getMessage());
+                            }
+                        } else {
+                            logger.info("{} - {} - entity data NOT found", uuid, dateString);
+                        }
                     }
                 } catch (Exception e) {
                     logger.error("{} - exception while trying to load entity data, {} ", uuid, e.getMessage());


[26/35] usergrid git commit: Allow GET /management/me with local UG tokens when External SSO is enabled.

Posted by mr...@apache.org.
Allow GET /management/me with local UG tokens when External SSO is enabled.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f73ce4a7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f73ce4a7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f73ce4a7

Branch: refs/heads/master
Commit: f73ce4a72696eac4db0b45ccb6e64a6f6f758494
Parents: 23cfde0
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Aug 31 11:46:21 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Aug 31 11:46:21 2016 -0700

----------------------------------------------------------------------
 .../rest/management/ManagementResource.java     | 21 ++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/f73ce4a7/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 4f8b456..12e1270 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -42,6 +42,7 @@ import org.apache.usergrid.security.sso.ApigeeSSO2Provider;
 import org.apache.usergrid.security.sso.ExternalSSOProvider;
 import org.apache.usergrid.security.sso.SSOProviderFactory;
 import org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl;
+import org.apache.usergrid.security.tokens.exceptions.BadTokenException;
 import org.apache.usergrid.utils.JsonUtils;
 import org.glassfish.jersey.server.mvc.Viewable;
 import org.slf4j.Logger;
@@ -196,14 +197,22 @@ public class ManagementResource extends AbstractContextResource {
         String ssoUserId = null;
         if(ssoEnabled && !user.getUsername().equals(properties.getProperty(USERGRID_SYSADMIN_LOGIN_NAME))){
             ExternalSSOProvider provider = ssoProviderFactory.getProvider();
-            final Map<String, String> decodedTokenDetails = provider.getDecodedTokenDetails(access_token);
-            final String expiry = decodedTokenDetails.containsKey("expiry") ? decodedTokenDetails.get("expiry") : "0";
 
-            tokenTtl =
-                Long.valueOf(expiry) - System.currentTimeMillis()/1000;
+            try {
+                final Map<String, String> decodedTokenDetails = provider.getDecodedTokenDetails(access_token);
+                final String expiry = decodedTokenDetails.containsKey("expiry") ? decodedTokenDetails.get("expiry") : "0";
+
+                tokenTtl =
+                    Long.valueOf(expiry) - System.currentTimeMillis() / 1000;
+
+                if (provider instanceof ApigeeSSO2Provider) {
+                    ssoUserId = decodedTokenDetails.get("user_id");
+                }
+            }catch (BadTokenException e){
+
+                // even when SSO is enabled, this could be a local token
+                tokenTtl = tokens.getTokenInfo(access_token).getDuration();
 
-            if( provider instanceof ApigeeSSO2Provider ) {
-                ssoUserId = decodedTokenDetails.get("user_id");
             }
 
         }else{


[05/35] usergrid git commit: Merge branch 'hotfix-20160819' of https://git-wip-us.apache.org/repos/asf/usergrid into hotfix-20160819

Posted by mr...@apache.org.
Merge branch 'hotfix-20160819' of https://git-wip-us.apache.org/repos/asf/usergrid into hotfix-20160819


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/911da7fa
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/911da7fa
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/911da7fa

Branch: refs/heads/master
Commit: 911da7fa5e3563e330ca98b9b35b0feaa944c936
Parents: 186110d a9c7190
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 23 18:47:49 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 23 18:47:49 2016 -0700

----------------------------------------------------------------------
 .../usergrid/services/users/UsersService.java   | 29 +-------------------
 1 file changed, 1 insertion(+), 28 deletions(-)
----------------------------------------------------------------------



[06/35] usergrid git commit: Revert "Make application users service use same logic for get by name."

Posted by mr...@apache.org.
Revert "Make application users service use same logic for get by name."

This reverts commit a9c7190a4b853a564c02f07ba285678b88a24df2.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fe6bea2b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fe6bea2b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fe6bea2b

Branch: refs/heads/master
Commit: fe6bea2bce6fe57814d1d90eba25296a04469920
Parents: 911da7f
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Aug 24 11:07:18 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Aug 24 11:07:18 2016 -0700

----------------------------------------------------------------------
 .../usergrid/services/users/UsersService.java   | 29 +++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/fe6bea2b/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java b/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java
index 90d4fff..0539695 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java
@@ -79,7 +79,34 @@ public class UsersService extends AbstractCollectionService {
 
     @Override
     public ServiceResults getItemByName( ServiceContext context, String name ) throws Exception {
-        super.getItemByName(context, name);
+        String nameProperty = Schema.getDefaultSchema().aliasProperty( getEntityType() );
+
+        if ( nameProperty == null ) {
+            nameProperty = "name";
+        }
+
+        EntityRef entity = null;
+        Identifier id = Identifier.from( name );
+
+        if ( id != null ) {
+            entity = em.getUserByIdentifier( id );
+        }
+
+        if ( entity == null ) {
+            throw new ServiceResourceNotFoundException( context );
+        }
+
+        if ( !context.moreParameters() ) {
+            entity = em.get( entity );
+            entity = importEntity( context, ( Entity ) entity );
+        }
+
+        checkPermissionsForEntity( context, entity );
+
+        List<ServiceRequest> nextRequests = context.getNextServiceRequests( entity );
+
+        return new ServiceResults( this, context, ServiceResults.Type.COLLECTION, Results.fromRef( entity ), null,
+                nextRequests );
     }
 
 


[18/35] usergrid git commit: Allow admin users to get tokens using credentials when SSO providers other than Usergrid is enabled.

Posted by mr...@apache.org.
Allow admin users to get tokens using credentials when SSO providers other than Usergrid is enabled.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a3e8946f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a3e8946f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a3e8946f

Branch: refs/heads/master
Commit: a3e8946fa275253c47bce93a98c6cfe058d4ceb1
Parents: 4b01bc8
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Aug 30 13:42:51 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Aug 30 13:42:51 2016 -0700

----------------------------------------------------------------------
 .../usergrid/rest/management/ManagementResource.java    | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a3e8946f/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 9ef67c9..4f8b456 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -395,6 +395,7 @@ public class ManagementResource extends AbstractContextResource {
 
             //moved the check for sso enabled form MangementServiceImpl since was unable to get the current user there to check if its super user.
             if( tokens.isExternalSSOProviderEnabled()
+                && properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_SSO_PROVIDER).equalsIgnoreCase("usergrid")
                 && !userServiceAdmin(username) ){
                 OAuthResponse response =
                     OAuthResponse.errorResponse( SC_BAD_REQUEST ).setError( OAuthError.TokenResponse.INVALID_GRANT )
@@ -625,13 +626,14 @@ public class ManagementResource extends AbstractContextResource {
             return; // we only care about username/password auth
         }
 
-        if ( tokens.isExternalSSOProviderEnabled() ) {
-            // when external tokens enabled then only superuser can obtain an access token
-            if ( !userServiceAdmin(username)) {
-                // this guy is not the superuser
+        // when external tokens enabled with Usergrid provider then only superuser can obtain an access token
+        if ( tokens.isExternalSSOProviderEnabled()
+            && properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_SSO_PROVIDER).equalsIgnoreCase("usergrid")
+            && !userServiceAdmin(username) ) {
+
                 throw new IllegalArgumentException( "External SSO integration is enabled, admin users must login via provider: "+
                     properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_SSO_PROVIDER) );
-            }
+
         }
     }
 


[28/35] usergrid git commit: Don't shut down full actor system when the JVM exists ( this affects actors running on other hosts ).

Posted by mr...@apache.org.
Don't shut down full actor system when the JVM exists ( this affects actors running on other hosts ).


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4275db0c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4275db0c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4275db0c

Branch: refs/heads/master
Commit: 4275db0c7adba6e904b46bdd451d87e8277c2933
Parents: f73ce4a
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Aug 31 21:24:23 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Aug 31 21:24:23 2016 -0700

----------------------------------------------------------------------
 .../persistence/actorsystem/ActorSystemManagerImpl.java | 12 ++++++------
 .../java/org/apache/usergrid/rest/ShutdownListener.java |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/4275db0c/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index 8dcb550..3f00327 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -349,12 +349,12 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
             }
 
             // add a shutdown hook to clean all actor systems if the JVM exits without the servlet container knowing
-            Runtime.getRuntime().addShutdownHook(new Thread() {
-                @Override
-                public void run() {
-                    shutdownAll();
-                }
-            });
+//            Runtime.getRuntime().addShutdownHook(new Thread() {
+//                @Override
+//                public void run() {
+//                    shutdownAll();
+//                }
+//            });
 
         }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/4275db0c/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java
index f9f1653..f3707a7 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java
@@ -72,7 +72,7 @@ public class ShutdownListener implements ServletContextListener {
         ActorSystemManager actorSystemManager = injector.getInstance(ActorSystemManager.class);
 
         // stop the Akka actor system
-        actorSystemManager.shutdownAll();
+        //actorSystemManager.shutdownAll();
 
         boolean started = Boolean.parseBoolean(
             properties.getProperty(JobServiceBoostrap.START_SCHEDULER_PROP, "true"));


[12/35] usergrid git commit: Fix temp logging issue.

Posted by mr...@apache.org.
Fix temp logging issue.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bf34a288
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bf34a288
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bf34a288

Branch: refs/heads/master
Commit: bf34a288a9c316adeed73c3786c985ba6782335d
Parents: 42c5c4b
Author: Michael Russo <mr...@apigee.com>
Authored: Sat Aug 27 23:30:33 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sat Aug 27 23:30:33 2016 -0700

----------------------------------------------------------------------
 .../serialization/impl/shard/impl/ShardGroupDeletionImpl.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/bf34a288/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index 7d91eda..6259e02 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -219,7 +219,7 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
 
             result = DeleteResult.DELETED;
 
-            logger.info( "{} - Marking shard {} as deleted in group {}", Thread.currentThread().getName()shard, shardEntryGroup );
+            logger.info( "Marking shard {} as deleted in group {}", shard, shardEntryGroup );
         }
 
 


[09/35] usergrid git commit: add deletion of orphans to collection iterator

Posted by mr...@apache.org.
add deletion of orphans to collection iterator


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cda2879c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cda2879c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cda2879c

Branch: refs/heads/master
Commit: cda2879c03d64c5d30565fb037d5b6588a8b793c
Parents: fd7a75e
Author: Mike Dunker <md...@apigee.com>
Authored: Fri Aug 26 16:40:38 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Fri Aug 26 16:40:38 2016 -0700

----------------------------------------------------------------------
 .../usergrid/tools/CollectionIterator.java      | 107 ++++++++++++++-----
 1 file changed, 81 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/cda2879c/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
index 855c363..a682e44 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -20,31 +20,19 @@ package org.apache.usergrid.tools;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
 
-import com.google.common.base.*;
 import com.google.common.base.Optional;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import com.netflix.astyanax.model.Column;
-import com.netflix.astyanax.model.ConsistencyLevel;
-import com.netflix.astyanax.util.RangeBuilder;
 import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
-import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
 import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
 import org.apache.usergrid.corepersistence.service.CollectionSearch;
 import org.apache.usergrid.corepersistence.service.CollectionService;
 import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
-import org.apache.usergrid.persistence.collection.serialization.impl.*;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.persistence.model.entity.*;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.field.StringField;
 import org.apache.usergrid.persistence.schema.CollectionInfo;
 import org.apache.usergrid.utils.InflectionUtils;
+import static org.apache.commons.lang.StringUtils.isBlank;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -54,11 +42,6 @@ import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 
 
-import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily;
-import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
-import rx.*;
-
 import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
 
 
@@ -72,6 +55,14 @@ public class CollectionIterator extends ToolBase {
 
     private static final String REMOVE_CONNECTIONS_ARG = "removeConnections";
 
+    private static final String LATEST_TIMESTAMP_ARG = "latestTimestamp";
+
+    private static final String EARLIEST_TIMESTAMP_ARG = "earliestTimestamp";
+
+    private static final String SECONDS_IN_PAST_ARG = "secondsInPast";
+
+    private static final Long DEFAULT_SECONDS_IN_PAST = 60L * 60L; // hour
+
     private EntityManager em;
 
     @Override
@@ -95,11 +86,29 @@ public class CollectionIterator extends ToolBase {
         options.addOption( collectionOption );
 
         Option removeConnectionsOption =
-            OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg().isRequired( false ).withDescription( "remove orphaned connections" )
-                .create(REMOVE_CONNECTIONS_ARG);
+                OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg().isRequired( false ).withDescription( "remove orphaned connections" )
+                        .create(REMOVE_CONNECTIONS_ARG);
 
         options.addOption( removeConnectionsOption );
 
+        Option earliestTimestampOption =
+                OptionBuilder.withArgName(EARLIEST_TIMESTAMP_ARG).hasArg().isRequired( false ).withDescription( "earliest timestamp to delete" )
+                        .create(EARLIEST_TIMESTAMP_ARG);
+
+        options.addOption( earliestTimestampOption );
+
+        Option latestTimestampOption =
+                OptionBuilder.withArgName(LATEST_TIMESTAMP_ARG).hasArg().isRequired( false ).withDescription( "latest timestamp to delete" )
+                        .create(LATEST_TIMESTAMP_ARG);
+
+        options.addOption( latestTimestampOption );
+
+        Option secondsInPastOption =
+            OptionBuilder.withArgName(SECONDS_IN_PAST_ARG).hasArg().isRequired( false ).withDescription( "how many seconds old orphan must be to be deleted" )
+                .create(SECONDS_IN_PAST_ARG);
+
+        options.addOption( secondsInPastOption );
+
 
         return options;
     }
@@ -116,18 +125,62 @@ public class CollectionIterator extends ToolBase {
 
         startSpring();
 
-        if (line.getOptionValue(APPLICATION_ARG).isEmpty()) {
+        String applicationOption = line.getOptionValue(APPLICATION_ARG);
+        String entityTypeOption = line.getOptionValue(ENTITY_TYPE_ARG);
+        String removeConnectionsOption = line.getOptionValue(REMOVE_CONNECTIONS_ARG);
+        String earliestTimestampOption = line.getOptionValue(EARLIEST_TIMESTAMP_ARG);
+        String latestTimestampOption = line.getOptionValue(LATEST_TIMESTAMP_ARG);
+        String secondsInPastOption = line.getOptionValue(SECONDS_IN_PAST_ARG);
+
+        if (isBlank(applicationOption)) {
             throw new RuntimeException("Application ID not provided.");
         }
         final UUID app = UUID.fromString(line.getOptionValue(APPLICATION_ARG));
 
-        String removeOrphansOption = line.getOptionValue(REMOVE_CONNECTIONS_ARG);
-        final boolean removeOrphans = !removeOrphansOption.isEmpty() && removeOrphansOption.toLowerCase().equals("yes");
+        if (isBlank(entityTypeOption)) {
+            throw new RuntimeException("Entity type (singular collection name) not provided.");
+        }
+        String entityType = entityTypeOption;
+
+        final boolean removeOrphans = !isBlank(removeConnectionsOption) && removeConnectionsOption.toLowerCase().equals("yes");
+
+        if (!isBlank(secondsInPastOption) && !isBlank(latestTimestampOption)) {
+            throw new RuntimeException("Can't specify both latest timestamp and seconds in past options.");
+        }
 
-        String entityType = line.getOptionValue(ENTITY_TYPE_ARG);
+        long earliest = 0L;
+        if (!isBlank(earliestTimestampOption)) {
+            try {
+                earliest = Long.parseLong(earliestTimestampOption);
+            } catch (Exception e) {
+                throw new RuntimeException("Cannot convert earliest timestamp to long: " + earliestTimestampOption);
+            }
+        }
+        final long earliestTimestamp = earliest;
+
+        long currentTimestamp = System.currentTimeMillis();
+
+        // default to DEFAULT_SECONDS_IN_PAST
+        long latest = currentTimestamp - (DEFAULT_SECONDS_IN_PAST * 1000L);
+        if (!isBlank(latestTimestampOption)) {
+            try {
+                latest = Long.parseLong(latestTimestampOption);
+            } catch (Exception e) {
+                throw new RuntimeException("Cannot convert latest timestamp to long: " + latestTimestampOption);
+            }
+        } else if (!isBlank(secondsInPastOption)) {
+            try {
+                long secondsInPast = Long.parseLong(secondsInPastOption);
+                latest = currentTimestamp - (secondsInPast * 1000L);
+            } catch (Exception e) {
+                throw new RuntimeException("Cannot convert seconds in past to long: " + secondsInPastOption);
+            }
+        }
+        final long latestTimestamp = latest;
 
         logger.info("Starting Tool: CollectionIterator");
         logger.info("Orphans {} be deleted", removeOrphans ? "WILL" : "will not");
+        logger.info("Timestamp range {} to {}", Long.toString(earliestTimestamp), Long.toString(latestTimestamp));
         logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));
 
         em = emf.getEntityManager( app );
@@ -184,15 +237,17 @@ public class CollectionIterator extends ToolBase {
 
                         logger.info("{} - {} - entity data found", uuid, dateString);
                     }else{
-                        if (removeOrphans) {
+                        if (removeOrphans && timestamp >= earliestTimestamp && timestamp <= latestTimestamp) {
                             logger.info("{} - {} - entity data NOT found, REMOVING", uuid, dateString);
                             try {
                                 em.removeFromCollection(headEntity, collectionName, entityRef );
                             } catch (Exception e) {
                                 logger.error("{} - exception while trying to remove orphaned connection, {}", uuid, e.getMessage());
                             }
+                        } else if (removeOrphans) {
+                            logger.info("{} - {} ({}) - entity data NOT found, not removing because timestamp not in range", uuid, dateString, timestamp);
                         } else {
-                            logger.info("{} - {} - entity data NOT found", uuid, dateString);
+                            logger.info("{} - {} ({}) - entity data NOT found", uuid, dateString, timestamp);
                         }
                     }
                 } catch (Exception e) {


[30/35] usergrid git commit: Move shard logging statement to TRACE level.

Posted by mr...@apache.org.
Move shard logging statement to TRACE level.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/69cfbbd1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/69cfbbd1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/69cfbbd1

Branch: refs/heads/master
Commit: 69cfbbd156e06be4dd38571a8fa76147ac364e54
Parents: 058d33a
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Aug 31 21:45:42 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Aug 31 21:45:42 2016 -0700

----------------------------------------------------------------------
 .../impl/shard/impl/ShardedEdgeSerializationImpl.java            | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/69cfbbd1/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
index 822f238..c518e75 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
@@ -729,12 +729,12 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
                 final R rowKey = getRowKey( shard );
                 writeEdge( batch, columnFamily, scope, rowKey, column, shard, isDeleted );
 
-                if(getDirectedEdge() instanceof DirectedEdge){
+                if(logger.isTraceEnabled() && getDirectedEdge() instanceof DirectedEdge){
                     DirectedEdge directedEdge = (DirectedEdge) getDirectedEdge();
                     if( shard != null && shard.getShardEnd().isPresent()
                         && directedEdge.timestamp > shard.getShardEnd().get().timestamp){
 
-                        logger.warn("Writing edge past shard end for edge: {}, shard: {}", directedEdge, shard );
+                        logger.trace("Writing edge past shard end for edge: {}, shard: {}", directedEdge, shard );
 
                     }
                 }


[07/35] usergrid git commit: Add tool that will iterator a collection and print UUIDs.

Posted by mr...@apache.org.
Add tool that will iterator a collection and print UUIDs.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a7639d55
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a7639d55
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a7639d55

Branch: refs/heads/master
Commit: a7639d55336dd696ea804f166811d6db90225b86
Parents: fe6bea2
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Aug 25 14:31:24 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Aug 25 14:31:24 2016 -0700

----------------------------------------------------------------------
 .../usergrid/tools/CollectionIterator.java      | 189 +++++++++++++++++++
 .../org/apache/usergrid/tools/ExportApp.java    | 118 +-----------
 .../org/apache/usergrid/tools/ToolBase.java     |   2 +
 .../usergrid/tools/export/ExportConnection.java |  74 ++++++++
 .../usergrid/tools/export/ExportEntity.java     |  66 +++++++
 5 files changed, 337 insertions(+), 112 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
new file mode 100644
index 0000000..02188cb
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.*;
+import com.google.common.base.Optional;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.astyanax.util.RangeBuilder;
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
+import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
+import org.apache.usergrid.corepersistence.service.CollectionSearch;
+import org.apache.usergrid.corepersistence.service.CollectionService;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.serialization.impl.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.model.entity.*;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+
+import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
+import rx.*;
+
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
+
+
+public class CollectionIterator extends ToolBase {
+
+    private static final Logger logger = LoggerFactory.getLogger( CollectionIterator.class );
+
+    private static final String APPLICATION_ARG = "app";
+
+    private static final String ENTITY_TYPE_ARG = "entityType";
+
+    private EntityManager em;
+
+    @Override
+    @SuppressWarnings( "static-access" )
+    public Options createOptions() {
+
+
+        Options options = super.createOptions();
+
+
+        Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( true )
+            .withDescription( "application id" ).create( APPLICATION_ARG );
+
+
+        options.addOption( appOption );
+
+        Option collectionOption =
+            OptionBuilder.withArgName(ENTITY_TYPE_ARG).hasArg().isRequired( true ).withDescription( "collection name" )
+                .create(ENTITY_TYPE_ARG);
+
+        options.addOption( collectionOption );
+
+
+        return options;
+    }
+
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see
+     * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+     */
+    @Override
+    public void runTool( CommandLine line ) throws Exception {
+
+        startSpring();
+
+        UUID appToFilter = null;
+        if (!line.getOptionValue(APPLICATION_ARG).isEmpty()) {
+            appToFilter = UUID.fromString(line.getOptionValue(APPLICATION_ARG));
+        }
+
+        String entityType = line.getOptionValue(ENTITY_TYPE_ARG);
+
+        logger.info("Staring Tool: CollectionIterator");
+        logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));
+
+        em = emf.getEntityManager( appToFilter );
+        EntityRef headEntity = new SimpleEntityRef("application", appToFilter);
+
+        CollectionService collectionService = injector.getInstance(CollectionService.class);
+        String collectionName = InflectionUtils.pluralize(entityType);
+
+        Query query = new Query();
+        query.setCollection(collectionName);
+        query.setLimit(1000);
+
+        com.google.common.base.Optional<String> queryString = com.google.common.base.Optional.absent();
+
+        CollectionInfo collection = getDefaultSchema().getCollection(headEntity.getType(), collectionName);
+
+        final UUID app = appToFilter;
+
+
+        IdQueryExecutor idQueryExecutor = new IdQueryExecutor(query.getCursor()) {
+            @Override
+            protected rx.Observable<ResultsPage<Id>> buildNewResultsPage(
+                final Optional<String> cursor) {
+
+                final CollectionSearch search =
+                    new CollectionSearch(new ApplicationScopeImpl(new SimpleId(app, "application")), new SimpleId(app, "application"), collectionName, collection.getType(), query.getLimit(),
+                        queryString, cursor);
+
+                return collectionService.searchCollectionIds(search);
+            }
+        };
+
+        while ( idQueryExecutor.hasNext() ){
+
+            Results results = idQueryExecutor.next();
+
+            List<UUID> ids = results.getIds();
+
+            ids.forEach( uuid -> {
+
+                try {
+                    org.apache.usergrid.persistence.Entity retrieved = em.get(new SimpleEntityRef(entityType, uuid));
+
+                    long timestamp = 0;
+                    String dateString = "NOT TIME-BASED";
+                    if (UUIDUtils.isTimeBased(uuid)){
+                        timestamp = UUIDUtils.getTimestampInMillis(uuid);
+                        Date uuidDate = new Date(timestamp);
+                        DateFormat df = new SimpleDateFormat();
+                        df.setTimeZone(TimeZone.getTimeZone("GMT"));
+                        dateString = df.format(uuidDate) + " GMT";
+                    }
+
+                    if ( retrieved != null ){
+
+                        logger.info("{} - {} - entity data found", uuid, dateString);
+                    }else{
+                        logger.info("{} - {} - entity data NOT found", uuid, dateString);
+                    }
+                } catch (Exception e) {
+                    logger.error("{} - exception while trying to load entity data, {} ", uuid, e.getMessage());
+                }
+
+
+            });
+
+        }
+
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
index 985c5f0..db59521 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
@@ -24,6 +24,8 @@ import org.apache.usergrid.persistence.Entity;
 import org.apache.usergrid.persistence.EntityManager;
 import org.apache.usergrid.persistence.Query;
 import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.tools.export.ExportConnection;
+import org.apache.usergrid.tools.export.ExportEntity;
 import org.apache.usergrid.utils.StringUtils;
 import org.codehaus.jackson.JsonGenerator;
 import org.codehaus.jackson.map.ObjectMapper;
@@ -195,7 +197,7 @@ public class ExportApp extends ExportingToolBase {
     /**
      * Emits entities of collection.
      */
-    class EntityObservable implements rx.Observable.OnSubscribe<ExportEntity> {
+    private class EntityObservable implements rx.Observable.OnSubscribe<ExportEntity> {
         EntityManager em;
         String collection;
 
@@ -264,7 +266,7 @@ public class ExportApp extends ExportingToolBase {
     /**
      * Emits connections of an entity.
      */
-    class ConnectionsObservable implements rx.Observable.OnSubscribe<ExportConnection> {
+    private class ConnectionsObservable implements rx.Observable.OnSubscribe<ExportConnection> {
         EntityManager em;
         ExportEntity exportEntity;
 
@@ -330,7 +332,7 @@ public class ExportApp extends ExportingToolBase {
     /**
      * Writes entities to JSON file.
      */
-    class EntityWriteAction implements Action1<ExportEntity> {
+    private class EntityWriteAction implements Action1<ExportEntity> {
 
         public void call(ExportEntity entity) {
 
@@ -368,7 +370,7 @@ public class ExportApp extends ExportingToolBase {
     /**
      * Writes connection to JSON file.
      */
-    class ConnectionWriteAction implements Action1<ExportConnection> {
+    private class ConnectionWriteAction implements Action1<ExportConnection> {
 
         public void call(ExportConnection conn) {
 
@@ -432,111 +434,3 @@ public class ExportApp extends ExportingToolBase {
         }
     }
 }
-
-
-/**
- * Represents entity data to be serialized to JSON.
- */
-class ExportEntity {
-    private String organization;
-    private String application;
-    private Entity entity;
-    private Map<String, Object> dictionaries;
-    public ExportEntity( String organization, String application, Entity entity, Map<String, Object> dictionaries ) {
-        this.organization = organization;
-        this.application = application;
-        this.entity = entity;
-        this.dictionaries = dictionaries;
-    }
-
-    public String getApplication() {
-        return application;
-    }
-
-    public void setApplication(String application) {
-        this.application = application;
-    }
-
-    public Entity getEntity() {
-        return entity;
-    }
-
-    public void setEntity(Entity entity) {
-        this.entity = entity;
-    }
-
-    public Map<String, Object> getDictionaries() {
-        return dictionaries;
-    }
-
-    public void setDictionaries(Map<String, Object> dictionaries) {
-        this.dictionaries = dictionaries;
-    }
-
-    public String getOrganization() {
-        return organization;
-    }
-
-    public void setOrganization(String organization) {
-        this.organization = organization;
-    }
-}
-
-
-/**
- * Represents connection data to be serialized to JSON.
- */
-class ExportConnection {
-    private String organization;
-    private String application;
-    private String connectionType;
-    private UUID sourceUuid;
-    private UUID targetUuid;
-    public ExportConnection(String organization, String application, String connectionType, UUID sourceUuid, UUID targetUuid) {
-        this.organization= organization;
-        this.application = application;
-        this.connectionType = connectionType;
-        this.sourceUuid = sourceUuid;
-        this.targetUuid = targetUuid;
-    }
-
-    public String getApplication() {
-        return application;
-    }
-
-    public void setApplication(String application) {
-        this.application = application;
-    }
-
-    public String getConnectionType() {
-        return connectionType;
-    }
-
-    public void setConnectionType(String connectionType) {
-        this.connectionType = connectionType;
-    }
-
-    public UUID getSourceUuid() {
-        return sourceUuid;
-    }
-
-    public void setSourceUuid(UUID sourceUuid) {
-        this.sourceUuid = sourceUuid;
-    }
-
-    public UUID getTargetUuid() {
-        return targetUuid;
-    }
-
-    public void setTargetUuid(UUID targetUuid) {
-        this.targetUuid = targetUuid;
-    }
-
-    public String getOrganization() {
-        return organization;
-    }
-
-    public void setOrganization(String organization) {
-        this.organization = organization;
-    }
-}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
index e82e5f8..c33c5f5 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
@@ -99,6 +99,8 @@ public abstract class ToolBase {
 
         //job scheduler also not needed for tools
         System.setProperty("usergrid.scheduler.enabled", "false");
+        System.setProperty("usergrid.cluster.enabled", "false");
+
 
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java
new file mode 100644
index 0000000..18b4a26
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportConnection.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools.export;
+
+import java.util.UUID;
+
+public class ExportConnection {
+    private String organization;
+    private String application;
+    private String connectionType;
+    private UUID sourceUuid;
+    private UUID targetUuid;
+    public ExportConnection(String organization, String application, String connectionType, UUID sourceUuid, UUID targetUuid) {
+        this.organization= organization;
+        this.application = application;
+        this.connectionType = connectionType;
+        this.sourceUuid = sourceUuid;
+        this.targetUuid = targetUuid;
+    }
+
+    public String getApplication() {
+        return application;
+    }
+
+    public void setApplication(String application) {
+        this.application = application;
+    }
+
+    public String getConnectionType() {
+        return connectionType;
+    }
+
+    public void setConnectionType(String connectionType) {
+        this.connectionType = connectionType;
+    }
+
+    public UUID getSourceUuid() {
+        return sourceUuid;
+    }
+
+    public void setSourceUuid(UUID sourceUuid) {
+        this.sourceUuid = sourceUuid;
+    }
+
+    public UUID getTargetUuid() {
+        return targetUuid;
+    }
+
+    public void setTargetUuid(UUID targetUuid) {
+        this.targetUuid = targetUuid;
+    }
+
+    public String getOrganization() {
+        return organization;
+    }
+
+    public void setOrganization(String organization) {
+        this.organization = organization;
+    }
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/a7639d55/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java
new file mode 100644
index 0000000..800287c
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/export/ExportEntity.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools.export;
+
+import org.apache.usergrid.persistence.Entity;
+
+import java.util.Map;
+
+public class ExportEntity {
+    private String organization;
+    private String application;
+    private Entity entity;
+    private Map<String, Object> dictionaries;
+    public ExportEntity( String organization, String application, Entity entity, Map<String, Object> dictionaries ) {
+        this.organization = organization;
+        this.application = application;
+        this.entity = entity;
+        this.dictionaries = dictionaries;
+    }
+
+    public String getApplication() {
+        return application;
+    }
+
+    public void setApplication(String application) {
+        this.application = application;
+    }
+
+    public Entity getEntity() {
+        return entity;
+    }
+
+    public void setEntity(Entity entity) {
+        this.entity = entity;
+    }
+
+    public Map<String, Object> getDictionaries() {
+        return dictionaries;
+    }
+
+    public void setDictionaries(Map<String, Object> dictionaries) {
+        this.dictionaries = dictionaries;
+    }
+
+    public String getOrganization() {
+        return organization;
+    }
+
+    public void setOrganization(String organization) {
+        this.organization = organization;
+    }
+}


[22/35] usergrid git commit: Make CollectionIterator tool use graph manager.

Posted by mr...@apache.org.
Make CollectionIterator tool use graph manager.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7408f3d5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7408f3d5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7408f3d5

Branch: refs/heads/master
Commit: 7408f3d56a05d57e62964eaa391146603a8099d5
Parents: 620e837
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Aug 30 16:05:51 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Aug 30 16:05:51 2016 -0700

----------------------------------------------------------------------
 .../usergrid/tools/CollectionIterator.java      | 40 +++++++-------------
 1 file changed, 14 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/7408f3d5/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
index 958f03d..bec844b 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -24,6 +24,7 @@ import java.util.*;
 import com.google.common.base.Optional;
 import com.netflix.astyanax.MutationBatch;
 import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
 import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
 import org.apache.usergrid.corepersistence.service.CollectionSearch;
 import org.apache.usergrid.corepersistence.service.CollectionService;
@@ -31,13 +32,14 @@ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.*;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.*;
 import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
 import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
 import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
 import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.persistence.model.entity.*;
+import org.apache.usergrid.persistence.model.entity.Entity;
 import org.apache.usergrid.persistence.schema.CollectionInfo;
 import org.apache.usergrid.utils.InflectionUtils;
 import static org.apache.commons.lang.StringUtils.isBlank;
@@ -212,29 +214,18 @@ public class CollectionIterator extends ToolBase {
 
         CollectionInfo collection = getDefaultSchema().getCollection(headEntity.getType(), collectionName);
 
+        GraphManagerFactory gmf = injector.getInstance( GraphManagerFactory.class );
+        GraphManager gm = gmf.createEdgeManager(applicationScope);
 
-        IdQueryExecutor idQueryExecutor = new IdQueryExecutor(query.getCursor()) {
-            @Override
-            protected rx.Observable<ResultsPage<Id>> buildNewResultsPage(
-                final Optional<String> cursor) {
 
-                final CollectionSearch search =
-                    new CollectionSearch(new ApplicationScopeImpl(new SimpleId(app, "application")), new SimpleId(app, "application"), collectionName, collection.getType(), query.getLimit(),
-                        queryString, cursor);
+        final SimpleSearchByEdgeType search =
+            new SimpleSearchByEdgeType( applicationScopeId, simpleEdgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+                Optional.absent(), false );
 
-                return collectionService.searchCollectionIds(search);
-            }
-        };
-
-        while ( idQueryExecutor.hasNext() ){
-
-            Results results = idQueryExecutor.next();
-
-            List<UUID> ids = results.getIds();
+        gm.loadEdgesFromSource(search).map(markedEdge -> {
 
-            ids.forEach( uuid -> {
-
-                try {
+            UUID uuid = markedEdge.getTargetNode().getUuid();
+            try {
                     EntityRef entityRef = new SimpleEntityRef(entityType, uuid);
                     org.apache.usergrid.persistence.Entity retrieved = em.get(entityRef);
 
@@ -257,8 +248,6 @@ public class CollectionIterator extends ToolBase {
                             try {
                                 //em.removeItemFromCollection(headEntity, collectionName, entityRef );
                                 logger.info("entityRef.asId(): {}", entityRef.asId());
-                                Edge edge = new SimpleEdge(applicationScopeId, simpleEdgeType, entityRef.asId(), timestamp);
-                                MarkedEdge markedEdge = new SimpleMarkedEdge(edge, true);
                                 MutationBatch batch = edgeSerialization.deleteEdge(applicationScope, markedEdge, UUIDUtils.newTimeUUID());
                                 logger.info("BATCH: {}", batch);
                                 batch.execute();
@@ -276,10 +265,9 @@ public class CollectionIterator extends ToolBase {
                 }
 
 
-            });
-
-        }
 
+           return markedEdge;
+        }).toBlocking().last();
 
     }
 }


[35/35] usergrid git commit: Merge branch 'hotfix-20160819'

Posted by mr...@apache.org.
Merge branch 'hotfix-20160819'


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/aa0353d9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/aa0353d9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/aa0353d9

Branch: refs/heads/master
Commit: aa0353d96716642746a8bc185414a11948138da0
Parents: ae39eca 9fae803
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Sep 1 17:51:09 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Sep 1 17:51:09 2016 -0700

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  21 +-
 .../pipeline/read/ResultsPage.java              |  14 +-
 .../read/traverse/AbstractReadGraphFilter.java  |   7 +
 .../results/IdQueryExecutor.java                |  10 +
 .../results/ObservableQueryExecutor.java        |   3 +
 .../actorsystem/ActorSystemManagerImpl.java     |  12 +-
 .../mvcc/stage/CollectionIoEvent.java           |  10 +-
 .../mvcc/stage/delete/MarkCommit.java           |   2 +-
 .../mvcc/stage/write/WriteCommit.java           |  13 +-
 .../mvcc/stage/write/WriteUniqueVerify.java     |  16 +-
 .../uniquevalues/UniqueValuesServiceTest.java   |   4 +-
 .../astyanax/MultiRowShardColumnIterator.java   |  30 ++
 .../persistence/core/shard/SmartShard.java      |   9 +-
 .../serialization/impl/shard/DirectedEdge.java  |   7 +
 .../impl/shard/NodeShardAllocation.java         |   5 +-
 .../graph/serialization/impl/shard/Shard.java   |  17 +-
 .../impl/shard/impl/EdgeSearcher.java           |  17 +-
 .../shard/impl/NodeShardAllocationImpl.java     |  13 +-
 .../impl/shard/impl/NodeShardCacheImpl.java     |  24 +-
 .../shard/impl/ShardGroupCompactionImpl.java    |   2 +-
 .../impl/shard/impl/ShardGroupDeletionImpl.java |  24 +-
 .../impl/ShardedEdgeSerializationImpl.java      | 116 ++++--
 .../shard/impl/serialize/ShardSerializer.java   |  11 +-
 .../graph/GraphManagerShardConsistencyIT.java   | 396 +++++++++++++++++--
 .../impl/shard/NodeShardAllocationTest.java     |   4 +-
 .../impl/shard/NodeShardCacheTest.java          |   4 +-
 .../persistence/index/query/Identifier.java     |   2 +-
 .../apache/usergrid/rest/ShutdownListener.java  |   2 +-
 .../rest/applications/users/UsersResource.java  |  14 +-
 .../rest/management/ManagementResource.java     |  33 +-
 .../collection/CollectionsResourceIT.java       |  18 +-
 .../rest/management/ManagementResourceIT.java   |   4 +-
 .../usergrid/tools/CollectionIterator.java      | 273 +++++++++++++
 .../org/apache/usergrid/tools/ExportApp.java    | 118 +-----
 .../org/apache/usergrid/tools/ShardManager.java | 242 ++++++++++++
 .../org/apache/usergrid/tools/ToolBase.java     |   2 +
 .../usergrid/tools/UniqueValueManager.java      | 311 +++++++++++++++
 .../usergrid/tools/export/ExportConnection.java |  74 ++++
 .../usergrid/tools/export/ExportEntity.java     |  66 ++++
 39 files changed, 1678 insertions(+), 272 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa0353d9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index fc1382f,9d376f9..edd44e8
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@@ -22,10 -22,7 +22,11 @@@ import java.util.HashMap
  import java.util.Map;
  import java.util.UUID;
  
 +
 +import com.datastax.driver.core.BatchStatement;
 +import com.datastax.driver.core.Session;
 +
+ import org.apache.commons.lang3.StringUtils;
  import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
  import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
  import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa0353d9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index e7dbb10,c51c742..7ef573d
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@@ -30,9 -22,14 +30,10 @@@ import com.google.common.base.Precondit
  import com.google.inject.Inject;
  import com.google.inject.Singleton;
  import com.netflix.astyanax.Keyspace;
 -import com.netflix.astyanax.MutationBatch;
 -import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
 -import com.netflix.astyanax.model.ConsistencyLevel;
  import com.netflix.hystrix.HystrixCommand;
  import com.netflix.hystrix.HystrixCommandGroupKey;
 -import com.netflix.hystrix.HystrixCommandProperties;
  import com.netflix.hystrix.HystrixThreadPoolProperties;
+ import org.apache.commons.lang3.StringUtils;
  import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
  import org.apache.usergrid.persistence.collection.MvccEntity;
  import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
@@@ -176,35 -169,40 +177,34 @@@ public class WriteUniqueVerify implemen
              // use write-first then read strategy
              final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
  
 -            try {
  
 -                // don't use read repair on this pre-write check
 -                // stronger consistency is extremely important here, more so than performance
 -                UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getReadCL(),
 -                    written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
 +            // don't use read repair on this pre-write check
-             // stronger consistency is extremely important here, more so than performance
-             UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getDataStaxReadConsistentCl(),
++            UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getDataStaxReadCl(),
 +                written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
  
 -                set.forEach(uniqueValue -> {
 +            set.forEach(uniqueValue -> {
  
 -                    if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
 -
 -                        if(logger.isTraceEnabled()){
 -                            logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and " +
 -                                "entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " +
 -                                "entity version [{}]",
 -                                written.getField().getName(),
 -                                written.getField().getValue().toString(),
 -                                written.getEntityId().getUuid(),
 -                                written.getEntityVersion(),
 -                                uniqueValue.getEntityId().getUuid(),
 -                                uniqueValue.getEntityVersion());
 -                        }
 -
 -                        preWriteUniquenessViolations.put(field.getName(), field);
 +                if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
  
 +                    if(logger.isTraceEnabled()){
 +                        logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and " +
 +                            "entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " +
 +                            "entity version [{}]",
 +                            written.getField().getName(),
 +                            written.getField().getValue().toString(),
 +                            written.getEntityId().getUuid(),
 +                            written.getEntityVersion(),
 +                            uniqueValue.getEntityId().getUuid(),
 +                            uniqueValue.getEntityVersion());
                      }
  
 -                });
 +                    preWriteUniquenessViolations.put(field.getName(), field);
  
 +                }
 +
 +            });
  
 -            } catch (ConnectionException e) {
  
 -                throw new RuntimeException("Error connecting to cassandra", e);
 -            }
  
              // only build the batch statement if we don't have a violation for the field
              if( preWriteUniquenessViolations.get(field.getName()) == null) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa0353d9/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa0353d9/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
----------------------------------------------------------------------


[32/35] usergrid git commit: Set non-clustered unique value verification loading back to the non-consistent read consistency level.

Posted by mr...@apache.org.
Set non-clustered unique value verification loading back to the non-consistent read consistency level.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f4b7ac6c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f4b7ac6c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f4b7ac6c

Branch: refs/heads/master
Commit: f4b7ac6c1494f45eff39ab003c18bf58c944235c
Parents: 9bb0ae6
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Sep 1 11:06:36 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Sep 1 11:06:36 2016 -0700

----------------------------------------------------------------------
 .../persistence/collection/mvcc/stage/write/WriteUniqueVerify.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/f4b7ac6c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 7a63061..0832074 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -173,7 +173,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
 
                 // don't use read repair on this pre-write check
                 // stronger consistency is extremely important here, more so than performance
-                UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getConsistentReadCL(),
+                UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getReadCL(),
                     written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
 
                 set.forEach(uniqueValue -> {


[17/35] usergrid git commit: Make sure root shard cannot be set to compacted=false in ShardManager tool.

Posted by mr...@apache.org.
Make sure root shard cannot be set to compacted=false in ShardManager tool.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b01bc88
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b01bc88
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b01bc88

Branch: refs/heads/master
Commit: 4b01bc88960db9008a553871a9f9a526baf09e45
Parents: 09bcce5
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Aug 29 17:11:01 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Aug 29 17:11:01 2016 -0700

----------------------------------------------------------------------
 .../org/apache/usergrid/tools/ShardManager.java | 36 +++++++++++++++++---
 1 file changed, 32 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b01bc88/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java
index ecf447c..96320c9 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ShardManager.java
@@ -57,6 +57,8 @@ public class ShardManager extends ToolBase {
 
     private static final String SHARD_TYPE_ARG = "shardType";
 
+    private static final String SHARD_INDEX_ARG = "shardIndex";
+
 
     @Override
     @SuppressWarnings("static-access")
@@ -84,11 +86,17 @@ public class ShardManager extends ToolBase {
         options.addOption(repairOption);
 
         Option shardTypeOption =
-            OptionBuilder.withArgName(SHARD_TYPE_ARG).hasArg().isRequired(true).withDescription("either collection or connection")
+            OptionBuilder.withArgName(SHARD_TYPE_ARG).hasArg().isRequired(false).withDescription("either collection or connection")
                 .create(SHARD_TYPE_ARG);
 
         options.addOption(shardTypeOption);
 
+        Option shardIndexOption =
+            OptionBuilder.withArgName(SHARD_INDEX_ARG).hasArg().isRequired(false).withDescription("the shardIndex")
+                .create(SHARD_INDEX_ARG);
+
+        options.addOption(shardIndexOption);
+
         return options;
     }
 
@@ -115,10 +123,13 @@ public class ShardManager extends ToolBase {
 
         String shardType = line.getOptionValue(SHARD_TYPE_ARG);
 
+        String shardIndex = line.getOptionValue(SHARD_INDEX_ARG);
+
         boolean repair = false;
         if( isNotEmpty(repairTask) && (
             repairTask.equalsIgnoreCase("removeAllShardEnds") || repairTask.equalsIgnoreCase("removeLastShardEnd") ||
-            repairTask.equalsIgnoreCase("resetAllCompactionStatus"))) {
+            repairTask.equalsIgnoreCase("resetAllCompactionStatus") || repairTask.equalsIgnoreCase("setCompacted") ||
+                repairTask.equalsIgnoreCase("clearCompacted"))) {
 
             repair = true;
         }
@@ -156,13 +167,30 @@ public class ShardManager extends ToolBase {
         while (shards.hasNext()) {
             Shard shard = shards.next();
 
+
             logger.info("Seeking over shard: {}", shard);
 
             if(repair) {
 
                 logger.info("Repair enabled with task: {}", repairTask);
 
-                if( repairTask.equalsIgnoreCase("removeLastShardEnd") && firstShard){
+                if ( repairTask.equalsIgnoreCase("setCompacted") && isNotEmpty(shardIndex)  && Long.valueOf(shardIndex).equals(shard.getShardIndex())){
+
+                    logger.info("Setting compacted=true for shard: {}", shard);
+
+                    shard.setCompacted(true);
+                    edgeShardSerialization.writeShardMeta(applicationScope, shard, directedEdgeMeta).execute();
+
+
+                }else if ( repairTask.equalsIgnoreCase("clearCompacted") && isNotEmpty(shardIndex)  && Long.valueOf(shardIndex).equals(shard.getShardIndex())){
+
+                    logger.info("Setting compacted=false for shard: {}", shard);
+
+                    shard.setCompacted(false);
+                    edgeShardSerialization.writeShardMeta(applicationScope, shard, directedEdgeMeta).execute();
+
+
+                } else if( repairTask.equalsIgnoreCase("removeLastShardEnd") && firstShard){
 
                     logger.info("Removing shard end from shard: {}", shard);
 
@@ -176,7 +204,7 @@ public class ShardManager extends ToolBase {
                     shard.setShardEnd(Optional.absent());
                     edgeShardSerialization.writeShardMeta(applicationScope, shard, directedEdgeMeta).execute();
 
-                } else if ( repairTask.equalsIgnoreCase("resetAllCompactionStatus")){
+                } else if ( repairTask.equalsIgnoreCase("resetAllCompactionStatus") && shard.getShardIndex() != 0){
 
                     logger.info("Setting compacted=false for shard: {}", shard);
 


[19/35] usergrid git commit: fix for CollectionIterator tool orphaned connections deletion

Posted by mr...@apache.org.
fix for CollectionIterator tool orphaned connections deletion


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0e0f8058
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0e0f8058
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0e0f8058

Branch: refs/heads/master
Commit: 0e0f8058074485b94c1690a28ed8990bcb6d2066
Parents: 4b01bc8
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 30 13:45:09 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 30 13:45:09 2016 -0700

----------------------------------------------------------------------
 .../apache/usergrid/tools/CollectionIterator.java    | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/0e0f8058/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
index a682e44..16c208d 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -22,12 +22,19 @@ import java.text.SimpleDateFormat;
 import java.util.*;
 
 import com.google.common.base.Optional;
+import com.netflix.astyanax.MutationBatch;
 import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
 import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
 import org.apache.usergrid.corepersistence.service.CollectionSearch;
 import org.apache.usergrid.corepersistence.service.CollectionService;
 import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
+import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
 import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.persistence.model.entity.*;
 import org.apache.usergrid.persistence.schema.CollectionInfo;
@@ -189,6 +196,9 @@ public class CollectionIterator extends ToolBase {
         CollectionService collectionService = injector.getInstance(CollectionService.class);
         String collectionName = InflectionUtils.pluralize(entityType);
 
+        ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(app, "application"));
+        EdgeSerialization edgeSerialization = injector.getInstance(EdgeSerialization.class);
+
         Query query = new Query();
         query.setCollection(collectionName);
         query.setLimit(1000);
@@ -240,7 +250,10 @@ public class CollectionIterator extends ToolBase {
                         if (removeOrphans && timestamp >= earliestTimestamp && timestamp <= latestTimestamp) {
                             logger.info("{} - {} - entity data NOT found, REMOVING", uuid, dateString);
                             try {
-                                em.removeFromCollection(headEntity, collectionName, entityRef );
+                                //em.removeItemFromCollection(headEntity, collectionName, entityRef );
+                                Edge edge = new SimpleEdge(applicationScope.getApplication(), entityType, entityRef.asId(), timestamp);
+                                MarkedEdge markedEdge = new SimpleMarkedEdge(edge, true);
+                                edgeSerialization.deleteEdge(applicationScope, markedEdge, uuid).execute();
                             } catch (Exception e) {
                                 logger.error("{} - exception while trying to remove orphaned connection, {}", uuid, e.getMessage());
                             }


[10/35] usergrid git commit: Add additional logging statements.

Posted by mr...@apache.org.
Add additional logging statements.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/dbdd2431
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/dbdd2431
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/dbdd2431

Branch: refs/heads/master
Commit: dbdd2431a7b65f6f2f8383e6c5fd598cac768329
Parents: fd7a75e
Author: Michael Russo <mr...@apigee.com>
Authored: Sat Aug 27 15:35:44 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sat Aug 27 15:35:44 2016 -0700

----------------------------------------------------------------------
 .../corepersistence/pipeline/read/ResultsPage.java    | 14 +++++++++++++-
 .../read/traverse/AbstractReadGraphFilter.java        |  7 +++++++
 .../corepersistence/results/IdQueryExecutor.java      | 10 ++++++++++
 .../results/ObservableQueryExecutor.java              |  3 +++
 4 files changed, 33 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/dbdd2431/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java
index 6b3a086..e4c3828 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java
@@ -23,7 +23,9 @@ package org.apache.usergrid.corepersistence.pipeline.read;
 import java.util.List;
 
 import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
-import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.corepersistence.results.ObservableQueryExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -34,6 +36,9 @@ import org.apache.usergrid.persistence.model.entity.Entity;
  */
 public class ResultsPage<T> {
 
+    private static final Logger logger = LoggerFactory.getLogger( ResultsPage.class );
+
+
     private final List<T> entityList;
 
     private final int limit;
@@ -57,6 +62,13 @@ public class ResultsPage<T> {
      * Return true if the results page is empty
      */
     public boolean hasMoreResults() {
+        if(logger.isTraceEnabled()){
+            if(entityList != null && entityList.size() == limit){
+                logger.trace("hasMoreResults = true, entityList size: {}, limit: {}", entityList.size(), limit);
+            }else{
+                logger.trace("hasMoreResults = false, entityList size: {}, limit: {}", entityList!= null ? entityList.size() : "null", limit);
+            }
+        }
         return entityList != null && entityList.size() == limit;
     }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/dbdd2431/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index f2aed89..62f6548 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -198,9 +198,16 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
 
         //if it's our first pass, there's no cursor to generate
         if(cursorValue == null){
+            if(logger.isTraceEnabled()){
+                logger.trace("Cursor value is null, creating filter result with no cursor");
+            }
             return new FilterResult<>( emit, parent );
         }
 
+        if(logger.isTraceEnabled()){
+            logger.trace("Cursor value is not null, creating filter result with cursor: {}", cursorValue.toString());
+        }
+
         return super.createFilterResult( emit, cursorValue, parent );
     }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/dbdd2431/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java
index 5a0bb3f..ea3d0ac 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java
@@ -36,6 +36,8 @@ import org.apache.usergrid.persistence.model.entity.Id;
 
 import com.google.common.base.Optional;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import rx.Observable;
 
 
@@ -45,6 +47,9 @@ import rx.Observable;
 @Deprecated//Required for 1.0 compatibility
 public abstract class IdQueryExecutor extends ObservableQueryExecutor<Id> {
 
+    private static final Logger logger = LoggerFactory.getLogger( IdQueryExecutor.class );
+
+
 
     protected IdQueryExecutor( final Optional<String> startCursor ) {
         super( startCursor );
@@ -53,6 +58,11 @@ public abstract class IdQueryExecutor extends ObservableQueryExecutor<Id> {
 
     @Override
     protected Results createResults( final ResultsPage resultsPage ) {
+
+        if(logger.isTraceEnabled()){
+            logger.trace("Creating Id results from resultsPage");
+        }
+
         final List<Id> ids = resultsPage.getEntityList();
 
         List<UUID> uuids = ids.stream().map(id -> id.getUuid()).collect(Collectors.toList());

http://git-wip-us.apache.org/repos/asf/usergrid/blob/dbdd2431/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
index 007486f..e12178f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
@@ -107,6 +107,9 @@ public abstract class ObservableQueryExecutor<T> implements QueryExecutor {
     public boolean hasNext() {
 
         if ( !complete && results == null) {
+            if(logger.isTraceEnabled()){
+                logger.trace("Iterator not complete and there are results object is null, advancing");
+            }
             advance();
         }
 


[03/35] usergrid git commit: Make application users service use same logic for get by name.

Posted by mr...@apache.org.
Make application users service use same logic for get by name.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a9c7190a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a9c7190a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a9c7190a

Branch: refs/heads/master
Commit: a9c7190a4b853a564c02f07ba285678b88a24df2
Parents: d2cdcd3
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Aug 23 15:32:06 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Aug 23 15:32:06 2016 -0700

----------------------------------------------------------------------
 .../usergrid/services/users/UsersService.java   | 29 +-------------------
 1 file changed, 1 insertion(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9c7190a/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java b/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java
index 0539695..90d4fff 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/users/UsersService.java
@@ -79,34 +79,7 @@ public class UsersService extends AbstractCollectionService {
 
     @Override
     public ServiceResults getItemByName( ServiceContext context, String name ) throws Exception {
-        String nameProperty = Schema.getDefaultSchema().aliasProperty( getEntityType() );
-
-        if ( nameProperty == null ) {
-            nameProperty = "name";
-        }
-
-        EntityRef entity = null;
-        Identifier id = Identifier.from( name );
-
-        if ( id != null ) {
-            entity = em.getUserByIdentifier( id );
-        }
-
-        if ( entity == null ) {
-            throw new ServiceResourceNotFoundException( context );
-        }
-
-        if ( !context.moreParameters() ) {
-            entity = em.get( entity );
-            entity = importEntity( context, ( Entity ) entity );
-        }
-
-        checkPermissionsForEntity( context, entity );
-
-        List<ServiceRequest> nextRequests = context.getNextServiceRequests( entity );
-
-        return new ServiceResults( this, context, ServiceResults.Type.COLLECTION, Results.fromRef( entity ), null,
-                nextRequests );
+        super.getItemByName(context, name);
     }
 
 


[20/35] usergrid git commit: collection iterator

Posted by mr...@apache.org.
collection iterator


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/394958f7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/394958f7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/394958f7

Branch: refs/heads/master
Commit: 394958f7513d6e61879b9bb65dd3f598cf988ce5
Parents: 0e0f805
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 30 15:27:53 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 30 15:27:53 2016 -0700

----------------------------------------------------------------------
 .../org/apache/usergrid/tools/CollectionIterator.java   | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/394958f7/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
index 16c208d..958f03d 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
@@ -27,6 +27,7 @@ import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
 import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
 import org.apache.usergrid.corepersistence.service.CollectionSearch;
 import org.apache.usergrid.corepersistence.service.CollectionService;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.*;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
@@ -195,8 +196,12 @@ public class CollectionIterator extends ToolBase {
 
         CollectionService collectionService = injector.getInstance(CollectionService.class);
         String collectionName = InflectionUtils.pluralize(entityType);
+        String simpleEdgeType = CpNamingUtils.getEdgeTypeFromCollectionName(collectionName);
+        logger.info("simpleEdgeType: {}", simpleEdgeType);
 
         ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId(app, "application"));
+        Id applicationScopeId = applicationScope.getApplication();
+        logger.info("applicationScope.getApplication(): {}", applicationScopeId);
         EdgeSerialization edgeSerialization = injector.getInstance(EdgeSerialization.class);
 
         Query query = new Query();
@@ -251,9 +256,12 @@ public class CollectionIterator extends ToolBase {
                             logger.info("{} - {} - entity data NOT found, REMOVING", uuid, dateString);
                             try {
                                 //em.removeItemFromCollection(headEntity, collectionName, entityRef );
-                                Edge edge = new SimpleEdge(applicationScope.getApplication(), entityType, entityRef.asId(), timestamp);
+                                logger.info("entityRef.asId(): {}", entityRef.asId());
+                                Edge edge = new SimpleEdge(applicationScopeId, simpleEdgeType, entityRef.asId(), timestamp);
                                 MarkedEdge markedEdge = new SimpleMarkedEdge(edge, true);
-                                edgeSerialization.deleteEdge(applicationScope, markedEdge, uuid).execute();
+                                MutationBatch batch = edgeSerialization.deleteEdge(applicationScope, markedEdge, UUIDUtils.newTimeUUID());
+                                logger.info("BATCH: {}", batch);
+                                batch.execute();
                             } catch (Exception e) {
                                 logger.error("{} - exception while trying to remove orphaned connection, {}", uuid, e.getMessage());
                             }


[02/35] usergrid git commit: For GET /users/{id}, add new force query parameter to force user ID to be treated as a name (force=name) or email (force=email)

Posted by mr...@apache.org.
For GET /users/{id}, add new force query parameter to force user ID to be treated
as a name (force=name) or email (force=email)


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d2cdcd3f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d2cdcd3f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d2cdcd3f

Branch: refs/heads/master
Commit: d2cdcd3f87f5676dca0d9cd43f788226a1c686ea
Parents: ca49f1f
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 23 13:02:43 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 23 13:02:43 2016 -0700

----------------------------------------------------------------------
 .../rest/applications/users/UsersResource.java        | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2cdcd3f/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
index 247df8a..68a0a69 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
@@ -88,7 +88,7 @@ public class UsersResource extends ServiceResource {
 
     @Override
     @Path("{itemName}")
-    public AbstractContextResource addNameParameter( @Context UriInfo ui, @PathParam("itemName") PathSegment itemName )
+    public AbstractContextResource addNameParameter( @Context UriInfo ui, @PathParam("itemName") PathSegment itemName)
             throws Exception {
 
         if(logger.isTraceEnabled()){
@@ -109,7 +109,17 @@ public class UsersResource extends ServiceResource {
         addParameter( getServiceParameters(), itemName.getPath() );
 
         addMatrixParams( getServiceParameters(), ui, itemName );
-        Identifier id = Identifier.from( itemName.getPath() );
+
+        String forceString = ui.getQueryParameters().getFirst("force");
+
+        Identifier id;
+        if (forceString != null && "email".equals(forceString.toLowerCase())) {
+            id = Identifier.fromEmail(itemName.getPath().toLowerCase());
+        } else if (forceString != null && "name".equals(forceString.toLowerCase())) {
+            id = Identifier.fromName(itemName.getPath().toLowerCase());
+        } else {
+            id = Identifier.from(itemName.getPath());
+        }
         if ( id == null ) {
             throw new IllegalArgumentException( "Not a valid user identifier: " + itemName.getPath() );
         }


[21/35] usergrid git commit: Merge branch 'hotfix-20160819' of https://git-wip-us.apache.org/repos/asf/usergrid into hotfix-20160819

Posted by mr...@apache.org.
Merge branch 'hotfix-20160819' of https://git-wip-us.apache.org/repos/asf/usergrid into hotfix-20160819


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/620e8374
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/620e8374
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/620e8374

Branch: refs/heads/master
Commit: 620e8374692321c86d53c4815b36a1b551c848a6
Parents: 394958f a3e8946
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 30 15:28:26 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 30 15:28:26 2016 -0700

----------------------------------------------------------------------
 .../usergrid/rest/management/ManagementResource.java    | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------



[14/35] usergrid git commit: Merge branch 'hotfix-20160819' of https://git-wip-us.apache.org/repos/asf/usergrid into hotfix-20160819

Posted by mr...@apache.org.
Merge branch 'hotfix-20160819' of https://git-wip-us.apache.org/repos/asf/usergrid into hotfix-20160819


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/44404248
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/44404248
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/44404248

Branch: refs/heads/master
Commit: 44404248c377aa2986d5db62a08f9f604d7f8f8c
Parents: 34b1697 cda2879
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Aug 28 11:20:44 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Aug 28 11:20:44 2016 -0700

----------------------------------------------------------------------
 .../usergrid/tools/CollectionIterator.java      | 107 ++++++++++++++-----
 1 file changed, 81 insertions(+), 26 deletions(-)
----------------------------------------------------------------------



[24/35] usergrid git commit: Fix test for SSO enabled Usergrid when generating admin tokens.

Posted by mr...@apache.org.
Fix test for SSO enabled Usergrid when generating admin tokens.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1e925c02
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1e925c02
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1e925c02

Branch: refs/heads/master
Commit: 1e925c023ebb70d777b326529659d72e6f104d9d
Parents: 2f21b31
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Aug 30 16:23:18 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Aug 30 16:23:18 2016 -0700

----------------------------------------------------------------------
 .../apache/usergrid/rest/management/ManagementResourceIT.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/1e925c02/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index 1da00d4..1a3eb1d 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -38,6 +38,7 @@ import javax.ws.rs.core.Response;
 import java.io.IOException;
 import java.util.*;
 
+import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_SSO_PROVIDER;
 import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_SSO_PROVIDER_URL;
 import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_SSO_ENABLED;
 import static org.apache.usergrid.utils.MapUtils.hashMap;
@@ -660,7 +661,7 @@ public class ManagementResourceIT extends AbstractRestIT {
 
 
     @Test
-    public void testSuperuserOnlyWhenValidateExternalTokensEnabled() throws Exception {
+    public void testSuperuserOnlyWhenValidateExternalTokensEnabledForUsergridProvider() throws Exception {
 
         // create an org and an admin user
 
@@ -673,6 +674,7 @@ public class ManagementResourceIT extends AbstractRestIT {
 
         String suToken = clientSetup.getSuperuserToken().getAccessToken();
         Map<String, String> props = new HashMap<String, String>();
+        props.put(USERGRID_EXTERNAL_SSO_PROVIDER, "usergrid");
         props.put(USERGRID_EXTERNAL_SSO_ENABLED, "true");
         props.put(USERGRID_EXTERNAL_SSO_PROVIDER_URL, getBaseURI().toURL().toExternalForm() );
         pathResource( "testproperties" ).post( props );


[25/35] usergrid git commit: Add additional logging if an edge is being written past the shard end timestamp.

Posted by mr...@apache.org.
Add additional logging if an edge is being written past the shard end timestamp.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/23cfde0b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/23cfde0b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/23cfde0b

Branch: refs/heads/master
Commit: 23cfde0bc36ab5d6af53d6c7beb1ddc4c08877d0
Parents: 1e925c0
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Aug 31 10:27:25 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Aug 31 10:27:25 2016 -0700

----------------------------------------------------------------------
 .../graph/serialization/impl/shard/DirectedEdge.java      |  7 +++++++
 .../impl/shard/impl/NodeShardAllocationImpl.java          |  4 ++--
 .../impl/shard/impl/ShardedEdgeSerializationImpl.java     | 10 ++++++++++
 3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/23cfde0b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdge.java
index 720c948..b8587a8 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdge.java
@@ -38,4 +38,11 @@ public class DirectedEdge {
         this.timestamp = timestamp;
         this.id = id;
     }
+
+    @Override
+    public String toString(){
+
+        return "DirectedEdge{ uuid="+id.getUuid()+", type="+id.getType()+", timestamp="+timestamp+" }";
+
+    }
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/23cfde0b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index cf9a51c..876cd2c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -296,8 +296,8 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
 
         for ( DirectedEdgeMeta.NodeMeta node : directedEdgeMeta.getNodes() ) {
 
-            //short circuit
-            if ( !isNew || node.getId().getUuid().version() > 2 ) {
+            //short circuit if not a type 1 time UUID
+            if ( !isNew || node.getId().getUuid().version() != 1 ) {
                 return false;
             }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/23cfde0b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
index 55eb172..822f238 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
@@ -729,6 +729,16 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
                 final R rowKey = getRowKey( shard );
                 writeEdge( batch, columnFamily, scope, rowKey, column, shard, isDeleted );
 
+                if(getDirectedEdge() instanceof DirectedEdge){
+                    DirectedEdge directedEdge = (DirectedEdge) getDirectedEdge();
+                    if( shard != null && shard.getShardEnd().isPresent()
+                        && directedEdge.timestamp > shard.getShardEnd().get().timestamp){
+
+                        logger.warn("Writing edge past shard end for edge: {}, shard: {}", directedEdge, shard );
+
+                    }
+                }
+
                 // if an edge is being written to this shard, un-delete it in case it was previously marked
                 // don't un-delete if the edge write is to actually remove an edge
                 // Usergrid allows entities to be written with a UUID generated from the past (time)


[33/35] usergrid git commit: Rename the cluster authoritative region parameters/methods to be more specific.

Posted by mr...@apache.org.
Rename the cluster authoritative region parameters/methods to be more specific.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c97a2004
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c97a2004
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c97a2004

Branch: refs/heads/master
Commit: c97a20041b5b7bc846648e19b549b7ecb5bf350b
Parents: f4b7ac6
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Sep 1 11:26:37 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Sep 1 11:26:37 2016 -0700

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        | 21 ++++++++++----------
 .../mvcc/stage/CollectionIoEvent.java           | 10 +++++-----
 .../mvcc/stage/delete/MarkCommit.java           |  2 +-
 .../mvcc/stage/write/WriteCommit.java           | 12 +++++------
 .../mvcc/stage/write/WriteUniqueVerify.java     | 12 +++++------
 5 files changed, 29 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c97a2004/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 7b273e4..87a8649 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -106,6 +106,7 @@ public class CpEntityManager implements EntityManager {
     public static final String APPLICATION_COLLECTION = "application.collection.";
     public static final String APPLICATION_ENTITIES = "application.entities";
     public static final long ONE_COUNT = 1L;
+    public static final String AUTHORITATIVE_REGION_SETTING = "authoritativeRegion";
 
     private final UUID applicationId;
     private final EntityManagerFig entityManagerFig;
@@ -522,7 +523,7 @@ public class CpEntityManager implements EntityManager {
 
         try {
 
-            String region = lookupRegionForType( entity.getType() );
+            String region = lookupAuthoritativeRegionForType( entity.getType() );
 
             cpEntity = ecm.write( cpEntity, region ).toBlocking().last();
 
@@ -625,7 +626,7 @@ public class CpEntityManager implements EntityManager {
         Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
 
         //Step 1 & 2 of delete
-        String region = this.lookupRegionForType( entityRef.getType() );
+        String region = this.lookupAuthoritativeRegionForType( entityRef.getType() );
         return ecm.mark( entityId, region ).mergeWith( gm.markNode( entityId, createGraphOperationTimestamp() ) );
 
     }
@@ -1138,7 +1139,7 @@ public class CpEntityManager implements EntityManager {
         Optional<Map<String, Object>> existingSettings =
             collectionSettings.getCollectionSettings( collectionName );
         if ( existingSettings.isPresent() ) {
-            region = existingSettings.get().get("region").toString();
+            region = existingSettings.get().get(AUTHORITATIVE_REGION_SETTING).toString();
         }
 
         //TODO: does this call and others like it need a graphite reporter?
@@ -1818,12 +1819,12 @@ public class CpEntityManager implements EntityManager {
         }
 
         // if region specified
-        Object region = newSettings.get("region");
+        Object region = newSettings.get(AUTHORITATIVE_REGION_SETTING);
         if ( region != null ) {
 
             // passing an empty string causes region to be removed from settings
             if ( region.toString().trim().isEmpty() ) {
-                updatedSettings.remove("region");
+                updatedSettings.remove(AUTHORITATIVE_REGION_SETTING);
 
             } else {
                 // make sure region is in the configured region list
@@ -1831,7 +1832,7 @@ public class CpEntityManager implements EntityManager {
                 if (!regionList.contains( region )) {
                     throw new NullArgumentException( "Region " + region + " not in region list" );
                 }
-                updatedSettings.put("region", region);
+                updatedSettings.put(AUTHORITATIVE_REGION_SETTING, region);
             }
         }
 
@@ -2827,7 +2828,7 @@ public class CpEntityManager implements EntityManager {
                     cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() );
             }
 
-            String region = lookupRegionForType( entity.getType() );
+            String region = lookupAuthoritativeRegionForType( entity.getType() );
 
             //this does the write so before adding to a collection everything already exists already.
             cpEntity = ecm.write( cpEntity, region ).toBlocking().last();
@@ -3138,7 +3139,7 @@ public class CpEntityManager implements EntityManager {
     }
 
 
-    private  String lookupRegionForType( String type ) {
+    private  String lookupAuthoritativeRegionForType(String type ) {
 
         String region = null;
         String collectionName = Schema.defaultCollectionName( type );
@@ -3151,8 +3152,8 @@ public class CpEntityManager implements EntityManager {
         Optional<Map<String, Object>> existingSettings =
             collectionSettings.getCollectionSettings( collectionName );
 
-        if ( existingSettings.isPresent() && existingSettings.get().get("region") != null ) {
-            region = existingSettings.get().get("region").toString();
+        if ( existingSettings.isPresent() && existingSettings.get().get(AUTHORITATIVE_REGION_SETTING) != null ) {
+            region = existingSettings.get().get(AUTHORITATIVE_REGION_SETTING).toString();
         }
 
         return region;

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c97a2004/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/CollectionIoEvent.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/CollectionIoEvent.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/CollectionIoEvent.java
index e0388de..4116f25 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/CollectionIoEvent.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/CollectionIoEvent.java
@@ -33,16 +33,16 @@ public class CollectionIoEvent<T> implements Serializable {
 
     final private T event;
 
-    final private String region;
+    final private String authoritativeRegion;
 
     public CollectionIoEvent( final ApplicationScope context, final T event ) {
         this( context, event, null );
     }
 
-    public CollectionIoEvent( final ApplicationScope context, final T event, String region ) {
+    public CollectionIoEvent( final ApplicationScope context, final T event, String authoritativeRegion ) {
         this.context = context;
         this.event = event;
-        this.region = region;
+        this.authoritativeRegion = authoritativeRegion;
     }
 
 
@@ -55,7 +55,7 @@ public class CollectionIoEvent<T> implements Serializable {
         return event;
     }
 
-    public String getRegion() {
-        return region;
+    public String getAuthoritativeRegion() {
+        return authoritativeRegion;
     }
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c97a2004/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 57d92e1..c79f072 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -131,7 +131,7 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
 //        // actorSystemFig may be null in testing
 //        if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
 //
-//            String region = idIoEvent.getRegion();
+//            String region = idIoEvent.getAuthoritativeRegion();
 //            if ( region == null ) {
 //                region = uniqueValuesFig.getAuthoritativeRegion();
 //            }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c97a2004/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index 553ac4c..9d376f9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@ -136,14 +136,14 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Collect
 
         // akkaFig may be null when this is called from JUnit tests
         if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
-            String region = ioEvent.getRegion();
-            if ( StringUtils.isEmpty(region) ) {
-                region = uniqueValuesFig.getAuthoritativeRegion();
+            String authoritativeRegion = ioEvent.getAuthoritativeRegion();
+            if ( StringUtils.isEmpty(authoritativeRegion) ) {
+                authoritativeRegion = uniqueValuesFig.getAuthoritativeRegion();
             }
-            if ( StringUtils.isEmpty(region) ) {
-                region = actorSystemFig.getRegionLocal();
+            if ( StringUtils.isEmpty(authoritativeRegion) ) {
+                authoritativeRegion = actorSystemFig.getRegionLocal();
             }
-            confirmUniqueFieldsAkka( mvccEntity, version, applicationScope, region );
+            confirmUniqueFieldsAkka( mvccEntity, version, applicationScope, authoritativeRegion );
         } else {
             confirmUniqueFields( mvccEntity, version, applicationScope, logMutation );
         }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c97a2004/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 0832074..c51c742 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -124,15 +124,15 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
 
         final ApplicationScope applicationScope = ioevent.getEntityCollection();
 
-        String region = ioevent.getRegion();
-        if ( StringUtils.isEmpty(region) ) {
-            region = uniqueValuesFig.getAuthoritativeRegion();
+        String authoritativeRegion = ioevent.getAuthoritativeRegion();
+        if ( StringUtils.isEmpty(authoritativeRegion) ) {
+            authoritativeRegion = uniqueValuesFig.getAuthoritativeRegion();
         }
-        if ( StringUtils.isEmpty(region) ) {
-            region = actorSystemFig.getRegionLocal();
+        if ( StringUtils.isEmpty(authoritativeRegion) ) {
+            authoritativeRegion = actorSystemFig.getRegionLocal();
         }
         try {
-            akkaUvService.reserveUniqueValues( applicationScope, entity, mvccEntity.getVersion(), region );
+            akkaUvService.reserveUniqueValues( applicationScope, entity, mvccEntity.getVersion(), authoritativeRegion );
 
         } catch (UniqueValueException e) {
             Map<String, Field> violations = new HashMap<>();


[16/35] usergrid git commit: Make sure we skip over deleted shards on iterator initialization.

Posted by mr...@apache.org.
Make sure we skip over deleted shards on iterator initialization.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/09bcce5c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/09bcce5c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/09bcce5c

Branch: refs/heads/master
Commit: 09bcce5cbde0ecee10c552fc19320e2ad4a7cff9
Parents: 1e6fb1e
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Aug 29 13:12:26 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Aug 29 13:12:26 2016 -0700

----------------------------------------------------------------------
 .../persistence/core/astyanax/MultiRowShardColumnIterator.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/09bcce5c/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
index 2446968..3dc2d67 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
@@ -242,7 +242,7 @@ public class MultiRowShardColumnIterator<R, C, T> implements Iterator<T> {
             }
 
             // skip over shards that are marked deleted
-            if( currentShard.isDeleted() && currentShardIterator.hasNext() ){
+            while ( currentShard.isDeleted() && currentShardIterator.hasNext() ){
 
                 if(logger.isTraceEnabled()){
                     logger.trace("Shard is marked deleted - {}", currentShard);


[29/35] usergrid git commit: Merge branch 'hotfix-20160819' of https://git-wip-us.apache.org/repos/asf/usergrid into hotfix-20160819

Posted by mr...@apache.org.
Merge branch 'hotfix-20160819' of https://git-wip-us.apache.org/repos/asf/usergrid into hotfix-20160819


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/058d33a8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/058d33a8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/058d33a8

Branch: refs/heads/master
Commit: 058d33a8962661c878b6eca0cfa0941dc9ca04cb
Parents: 4275db0 9520496
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Aug 31 21:24:59 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Aug 31 21:24:59 2016 -0700

----------------------------------------------------------------------
 .../collection/uniquevalues/UniqueValuesServiceTest.java         | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------



[34/35] usergrid git commit: Update test for collection level authoritative region setting.

Posted by mr...@apache.org.
Update test for collection level authoritative region setting.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9fae8037
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9fae8037
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9fae8037

Branch: refs/heads/master
Commit: 9fae8037a4b881e9c13a5a1f23f71dc34e950c40
Parents: c97a200
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Sep 1 12:05:14 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Sep 1 12:05:14 2016 -0700

----------------------------------------------------------------------
 .../collection/CollectionsResourceIT.java         | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9fae8037/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index f13ed46..d72054a 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -49,6 +49,8 @@ import static org.junit.Assert.*;
 
 public class CollectionsResourceIT extends AbstractRestIT {
 
+    private final String REGION_SETTING = "authoritativeRegion";
+
     private static final Logger log = LoggerFactory.getLogger( CollectionsResourceIT.class );
 
 
@@ -1007,7 +1009,7 @@ public class CollectionsResourceIT extends AbstractRestIT {
 
 
     @Test
-    public void testCollectionRegion() {
+    public void testCollectionAuthoritativeRegion() {
 
         // create collection with settings for index all
 
@@ -1022,13 +1024,13 @@ public class CollectionsResourceIT extends AbstractRestIT {
 
         Collection collection = app().collection( collectionName ).collection( "_settings" ).get();
         Map<String, Object> settings = (Map<String, Object>)collection.getResponse().getData();
-        assertNull( settings.get( "region" ));
+        assertNull( settings.get( REGION_SETTING ));
 
         // set collection region with bad region, expect error
 
         try {
             app().collection( collectionName ).collection( "_settings" )
-                .post( new Entity().chainPut( "region", "us-moon-1" ) );
+                .post( new Entity().chainPut(REGION_SETTING, "us-moon-1" ) );
             fail( "post should have failed");
 
         } catch ( BadRequestException expected ) {}
@@ -1036,26 +1038,26 @@ public class CollectionsResourceIT extends AbstractRestIT {
         // set collection region with good region
 
         app().collection( collectionName ).collection( "_settings" )
-            .post( new Entity().chainPut( "region", "us-east-1" ) );
+            .post( new Entity().chainPut( REGION_SETTING, "us-east-1" ) );
 
         // get collection settings see that we have a region
 
         collection = app().collection( collectionName ).collection( "_settings" ).get();
         settings = (Map<String, Object>)collection.getResponse().getData();
-        assertNotNull( settings.get( "region" ));
-        assertEquals( "us-east-1", settings.get( "region" ));
+        assertNotNull( settings.get( REGION_SETTING ));
+        assertEquals( "us-east-1", settings.get( REGION_SETTING ));
 
         // unset the collection region
 
         app().collection( collectionName ).collection( "_settings" )
-            .post( new Entity().chainPut( "region", "" ) );
+            .post( new Entity().chainPut( REGION_SETTING, "" ) );
         refreshIndex();
 
         // get collection settings, should see no region
 
         collection = app().collection( collectionName ).collection( "_settings" ).get();
         settings = (Map<String, Object>)collection.getResponse().getData();
-        assertNull( settings.get( "region" ));
+        assertNull( settings.get( REGION_SETTING ));
 
 
     }


[04/35] usergrid git commit: Added UniqueValueManager tool to get and delete unique values

Posted by mr...@apache.org.
Added UniqueValueManager tool to get and delete unique values


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/186110d2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/186110d2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/186110d2

Branch: refs/heads/master
Commit: 186110d25164520cefb535cd279e24f1e02362f2
Parents: d2cdcd3
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 23 18:46:58 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 23 18:46:58 2016 -0700

----------------------------------------------------------------------
 .../usergrid/tools/UniqueValueManager.java      | 311 +++++++++++++++++++
 1 file changed, 311 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/186110d2/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueManager.java
new file mode 100644
index 0000000..6131866
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueManager.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.astyanax.util.RangeBuilder;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+import org.apache.usergrid.persistence.collection.serialization.impl.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+
+import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
+
+
+
+public class UniqueValueManager extends ToolBase {
+
+    private static final Logger logger = LoggerFactory.getLogger( UniqueValueManager.class );
+
+    private static final String OPERATION_ARG = "op";
+
+    private static final String CONFIRM_DELETE_ARG = "confirmDelete";
+
+    private static final String FILEPATH_ARG = "file";
+
+
+
+    //copied shamelessly from unique value serialization strat.
+    private static final ScopedRowKeySerializer<TypeField> ROW_KEY_SER =
+        new ScopedRowKeySerializer<>( UniqueTypeFieldRowKeySerializer.get() );
+
+
+    private final EntityVersionSerializer ENTITY_VERSION_SER = new EntityVersionSerializer();
+
+    private final MultiTenantColumnFamily<ScopedRowKey<TypeField>, EntityVersion> CF_UNIQUE_VALUES =
+        new MultiTenantColumnFamily<>( "Unique_Values_V2", ROW_KEY_SER, ENTITY_VERSION_SER );
+
+    private com.netflix.astyanax.Keyspace keyspace;
+
+    private MvccEntitySerializationStrategy mvccEntitySerializationStrategy;
+
+    private UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+
+    private EntityManager em;
+
+    @Override
+    @SuppressWarnings( "static-access" )
+    public Options createOptions() {
+
+
+        Options options = super.createOptions();
+
+        Option opOption =
+                OptionBuilder.withArgName(OPERATION_ARG).hasArg().isRequired( false ).withDescription( "operation" )
+                        .create(OPERATION_ARG);
+
+        options.addOption( opOption );
+
+        Option confirmDeleteOption =
+                OptionBuilder.withArgName(CONFIRM_DELETE_ARG).isRequired( false ).withDescription( "confirm delete" )
+                        .create(CONFIRM_DELETE_ARG);
+
+        options.addOption( confirmDeleteOption );
+
+        Option filepathOption =
+            OptionBuilder.withArgName(FILEPATH_ARG).hasArg().isRequired( true )
+                .withDescription( "path to file containing UV info" ).create(FILEPATH_ARG);
+
+        options.addOption( filepathOption );
+
+        return options;
+    }
+
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see
+     * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+     */
+    @Override
+    public void runTool( CommandLine line ) throws Exception {
+
+        startSpring();
+
+        logger.info("Staring Tool: UniqueValueManager");
+        logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));
+
+        String operation = line.getOptionValue(OPERATION_ARG) != null ? line.getOptionValue(OPERATION_ARG) : "get";
+        boolean deleteOp = operation.toLowerCase().equals("delete");
+        if (deleteOp && !line.hasOption(CONFIRM_DELETE_ARG)) {
+            throw new RuntimeException("Must add confirmDelete option to use delete.");
+        }
+        String filepath = line.getOptionValue(FILEPATH_ARG);
+        if (filepath == null || filepath.isEmpty()) {
+            throw new RuntimeException("File is required -- should contain one row per entity formatted like " +
+                    "'{uuid}|{entityType}|{fieldType}|{fieldValue}'.  " +
+                    "Example: 'b9398e88-ef7f-11e5-9e41-0a2cb9e6caa9|user|email|baasadmins@apigee.com'");
+        }
+
+        keyspace = injector.getInstance(com.netflix.astyanax.Keyspace.class);
+        mvccEntitySerializationStrategy = injector.getInstance(MvccEntitySerializationStrategy.class);
+        uniqueValueSerializationStrategy = injector.getInstance(UniqueValueSerializationStrategy.class);
+
+        AtomicInteger count = new AtomicInteger(0);
+
+        File listFile = new File(filepath);
+
+        try (BufferedReader br = new BufferedReader(new FileReader(listFile))) {
+            String fileLine;
+            while ((fileLine = br.readLine()) != null) {
+                String[] valuesArray = fileLine.trim().split("\\|");
+                if (valuesArray.length != 4) {
+                    logger.info("Line: >"+fileLine+"<");
+                    throw new RuntimeException("Invalid file -- should contain one row per entity formatted like " +
+                            "'{uuid}|{entityType}|{fieldType}|{fieldValue}'.  " +
+                            "Example: 'b9398e88-ef7f-11e5-9e41-0a2cb9e6caa9|user|email|whatever@usergrid.com'");
+                }
+                UUID appUuid = UUID.fromString(valuesArray[0]);
+                String entityType = valuesArray[1];
+                String fieldType = valuesArray[2];
+                String fieldValue = valuesArray[3];
+
+                UniqueValueSet uniqueValueSet = uniqueValueSerializationStrategy.load(
+                        new ApplicationScopeImpl(new SimpleId(appUuid, "application")),
+                        ConsistencyLevel.valueOf(System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")), entityType,
+                        Collections.singletonList(new StringField(fieldType, fieldValue)), false);
+
+                StringBuilder stringBuilder = new StringBuilder();
+
+                stringBuilder.append("[");
+
+                uniqueValueSet.forEach(uniqueValue -> {
+
+
+                    String entry = "fieldName=" + uniqueValue.getField().getName() +
+                            ", fieldValue=" + uniqueValue.getField().getValue() +
+                            ", uuid=" + uniqueValue.getEntityId().getUuid() +
+                            ", type=" + uniqueValue.getEntityId().getType() +
+                            ", version=" + uniqueValue.getEntityVersion();
+                    stringBuilder.append("{").append(entry).append("},");
+                });
+
+                stringBuilder.deleteCharAt(stringBuilder.length() - 1);
+                stringBuilder.append("]");
+
+                logger.info("Returned unique value set from serialization load = {}", stringBuilder.toString());
+
+                if (deleteOp) {
+                    uniqueValueSet.forEach(uniqueValue -> {
+                        logger.info("DELETING UNIQUE VALUE");
+                        try {
+                            uniqueValueSerializationStrategy.delete(new ApplicationScopeImpl(new SimpleId(appUuid, "application")), uniqueValue).execute();
+                        }
+                        catch (Exception e) {
+                            logger.error("Exception thrown for UV delete: " + e.getMessage());
+                        }
+                    });
+                }
+            }
+        }
+    }
+
+        /*
+        } else {
+
+            logger.info("Running entity unique scanner only");
+
+
+            // scan through all unique values and log some info
+
+            Iterator<com.netflix.astyanax.model.Row<ScopedRowKey<TypeField>, EntityVersion>> rows = null;
+            try {
+
+                rows = keyspace.prepareQuery(CF_UNIQUE_VALUES)
+                    .setConsistencyLevel(ConsistencyLevel.valueOf(System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")))
+                    .getAllRows()
+                    .withColumnRange(new RangeBuilder().setLimit(1000).build())
+                    .execute().getResult().iterator();
+
+            } catch (ConnectionException e) {
+
+                logger.error("Error connecting to cassandra", e);
+            }
+
+
+            UUID finalAppToFilter = appToFilter;
+
+            if( rows != null) {
+                rows.forEachRemaining(row -> {
+
+                    count.incrementAndGet();
+
+                    if(count.get() % 1000 == 0 ){
+                        logger.info("Scanned {} rows in {}", count.get(), CF_UNIQUE_VALUES.getName());
+                    }
+
+                    final String fieldName = row.getKey().getKey().getField().getName();
+                    final String fieldValue = row.getKey().getKey().getField().getValue().toString();
+                    final String scopeType = row.getKey().getScope().getType();
+                    final UUID scopeUUID = row.getKey().getScope().getUuid();
+
+
+                    if (!fieldName.equalsIgnoreCase(fieldType) ||
+                        (finalAppToFilter != null && !finalAppToFilter.equals(scopeUUID))
+                        ) {
+                        // do nothing
+
+                    } else {
+
+
+                        // if we have more than 1 column, let's check for a duplicate
+                        if (row.getColumns() != null && row.getColumns().size() > 1) {
+
+                            final List<EntityVersion> values = new ArrayList<>(row.getColumns().size());
+
+                            Iterator<Column<EntityVersion>> columns = row.getColumns().iterator();
+                            columns.forEachRemaining(column -> {
+
+
+                                final EntityVersion entityVersion = column.getName();
+
+
+                                logger.trace(
+                                    scopeType + ": " + scopeUUID + ", " +
+                                        fieldName + ": " + fieldValue + ", " +
+                                        "entity type: " + entityVersion.getEntityId().getType() + ", " +
+                                        "entity uuid: " + entityVersion.getEntityId().getUuid()
+                                );
+
+
+                                if (entityType != null &&
+                                    entityVersion.getEntityId().getType().equalsIgnoreCase(entityType)
+                                    ) {
+
+                                    // add the first value into the list
+                                    if (values.size() == 0) {
+
+                                        values.add(entityVersion);
+
+
+                                    } else {
+
+                                        if (!values.get(0).getEntityId().getUuid().equals(entityVersion.getEntityId().getUuid())) {
+
+                                            values.add(entityVersion);
+
+                                            logger.error("Duplicate found for field [{}={}].  Entry 1: [{}], Entry 2: [{}]",
+                                                fieldName, fieldValue, values.get(0).getEntityId(), entityVersion.getEntityId());
+
+                                        }
+
+                                    }
+
+
+                                }
+
+                            });
+                        }
+                    }
+
+
+                });
+            }else{
+
+                logger.warn("No rows returned from table: {}", CF_UNIQUE_VALUES.getName());
+
+            }
+
+        }
+    }
+    */
+}


[11/35] usergrid git commit: Move to shard deletion being a mark plus read filtering strategy vs. deleting. This is so Usergrid can keep proper shard end times and 're-activate' a shard by flipping the deleted flag when past data is re-written to a coll

Posted by mr...@apache.org.
Move to shard deletion being a mark plus read filtering strategy vs. deleting.  This is so Usergrid can keep proper shard end times and 're-activate' a shard by flipping the deleted flag when past data is re-written to a collection.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/42c5c4b8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/42c5c4b8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/42c5c4b8

Branch: refs/heads/master
Commit: 42c5c4b8760b78ea1e8ef9b8d583a92f2a45e1d4
Parents: dbdd243
Author: Michael Russo <mr...@apigee.com>
Authored: Sat Aug 27 23:24:01 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sat Aug 27 23:24:01 2016 -0700

----------------------------------------------------------------------
 .../astyanax/MultiRowShardColumnIterator.java   |  27 ++
 .../persistence/core/shard/SmartShard.java      |   9 +-
 .../impl/shard/NodeShardAllocation.java         |   5 +-
 .../graph/serialization/impl/shard/Shard.java   |  17 +-
 .../impl/shard/impl/EdgeSearcher.java           |  17 +-
 .../shard/impl/NodeShardAllocationImpl.java     |   9 +-
 .../impl/shard/impl/NodeShardCacheImpl.java     |  24 +-
 .../shard/impl/ShardGroupCompactionImpl.java    |   2 +-
 .../impl/shard/impl/ShardGroupDeletionImpl.java |  24 +-
 .../impl/ShardedEdgeSerializationImpl.java      | 106 +++--
 .../shard/impl/serialize/ShardSerializer.java   |  11 +-
 .../graph/GraphManagerShardConsistencyIT.java   | 396 +++++++++++++++++--
 .../impl/shard/NodeShardAllocationTest.java     |   4 +-
 .../impl/shard/NodeShardCacheTest.java          |   4 +-
 14 files changed, 559 insertions(+), 96 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
index f3e8d4c..04266e1 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
@@ -127,6 +127,23 @@ public class MultiRowShardColumnIterator<R, C, T> implements Iterator<T> {
             // advance to the next shard
             currentShard = currentShardIterator.next();
 
+            // handle marked deleted shards
+            if( currentShard.isDeleted() && currentShardIterator.hasNext()){
+
+                if(logger.isTraceEnabled()) {
+                    logger.trace("Shard is marked deleted, advancing to next - {}", currentShard);
+                }
+
+                currentShard = currentShardIterator.next();
+            }else if ( currentShard.isDeleted() && !currentShardIterator.hasNext()){
+
+                if(logger.isTraceEnabled()) {
+                    logger.trace("Shard is marked deleted, and there is no more - {}", currentShard);
+                }
+
+                return false;
+            }
+
             if(logger.isTraceEnabled()){
                 logger.trace("Shard after advance: {}", currentShard);
 
@@ -221,6 +238,16 @@ public class MultiRowShardColumnIterator<R, C, T> implements Iterator<T> {
                 }
             }
 
+            // skip over shards that are marked deleted
+            if( currentShard.isDeleted() && currentShardIterator.hasNext() ){
+
+                if(logger.isTraceEnabled()){
+                    logger.trace("Shard is marked deleted - {}", currentShard);
+                }
+
+                currentShard = currentShardIterator.next();
+            }
+
 
             if(logger.isTraceEnabled()){
                 logger.trace("all shards when starting: {}", rowKeysWithShardEnd);

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/shard/SmartShard.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/shard/SmartShard.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/shard/SmartShard.java
index 39ddc35..dd6df34 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/shard/SmartShard.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/shard/SmartShard.java
@@ -25,13 +25,15 @@ public class SmartShard<R, T> {
     final ScopedRowKey<R> rowKey;
     final T shardEnd;
     final long shardIndex;
+    final boolean isDeleted;
 
 
-    public SmartShard(final ScopedRowKey<R> rowKey, final long shardIndex, final T shardEnd){
+    public SmartShard(final ScopedRowKey<R> rowKey, final long shardIndex, final T shardEnd, final boolean isDeleted){
 
         this.rowKey = rowKey;
         this.shardIndex = shardIndex;
         this.shardEnd = shardEnd;
+        this.isDeleted = isDeleted;
     }
 
 
@@ -48,12 +50,15 @@ public class SmartShard<R, T> {
         return shardIndex;
     }
 
+    public boolean isDeleted(){
+        return isDeleted;
+    }
 
 
     @Override
     public String toString(){
 
-        return "Shard { rowKey="+rowKey + ", shardEnd="+shardEnd+" }";
+        return "Shard { rowKey="+rowKey + ", shardIndex="+shardIndex+", shardEnd="+shardEnd+", isDeleted="+isDeleted+" }";
 
 
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocation.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocation.java
index fcc0fc3..49fde9b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocation.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocation.java
@@ -24,8 +24,6 @@ import java.util.Iterator;
 
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 
-import com.google.common.base.Optional;
-
 
 /**
  * Interface used to create and retrieve shards
@@ -38,11 +36,10 @@ public interface NodeShardAllocation {
      * Get all shards for the given info.  If none exist, a default shard should be allocated.  The nodeId is the source node
      *
      * @param scope The application scope
-     * @param maxShardId The max value to start seeking from.  Values <= this will be returned if specified
      * @param directedEdgeMeta The directed edge metadata to use
      * @return A list of all shards <= the current shard.  This will always return 0l if no shards are allocated
      */
-    public Iterator<ShardEntryGroup> getShards( final ApplicationScope scope, Optional<Shard> maxShardId, final DirectedEdgeMeta directedEdgeMeta );
+    public Iterator<ShardEntryGroup> getShards(final ApplicationScope scope, final DirectedEdgeMeta directedEdgeMeta);
 
 
     /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java
index 6394703..f92c37a 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java
@@ -32,8 +32,9 @@ public class Shard implements Comparable<Shard> {
 
     private final long shardIndex;
     private final long createdTime;
-    private final boolean compacted;
+    private boolean compacted;
     private Optional<DirectedEdge> shardEnd;
+    private boolean deleted;
 
 
     public Shard( final long shardIndex, final long createdTime, final boolean compacted ) {
@@ -41,6 +42,7 @@ public class Shard implements Comparable<Shard> {
         this.createdTime = createdTime;
         this.compacted = compacted;
         this.shardEnd = Optional.absent();
+        this.deleted = false;
     }
 
 
@@ -67,6 +69,10 @@ public class Shard implements Comparable<Shard> {
         return compacted;
     }
 
+    public void setCompacted(final boolean compacted){
+        this.compacted = compacted;
+    }
+
 
     /**
      * Returns true if this is the minimum shard
@@ -84,6 +90,14 @@ public class Shard implements Comparable<Shard> {
         return shardEnd;
     }
 
+    public boolean isDeleted(){
+        return deleted;
+    }
+
+    public void setDeleted( final boolean deleted){
+        this.deleted = deleted;
+    }
+
 
     /**
      * Compare the shards based on the timestamp first, then the created time second
@@ -174,6 +188,7 @@ public class Shard implements Comparable<Shard> {
         }else{
             string.append("null");
         }
+        string.append(", isDeleted=").append(deleted);
         string.append(" }");
 
         return string.toString();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
index 917e943..eb90866 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
@@ -80,6 +80,10 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Colum
 
     public List<ScopedRowKey<R>> getRowKeys() {
 
+        if(logger.isTraceEnabled()) {
+            logger.trace("Shards: {}", shards);
+        }
+
         List<ScopedRowKey<R>> rowKeys = new ArrayList<>(shards.size());
 
         for(Shard shard : shards){
@@ -89,12 +93,18 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Colum
             rowKeys.add( rowKey );
         }
 
+        if(logger.isTraceEnabled()) {
+            logger.trace("Resulting Shards: {}", rowKeys);
+        }
 
         return rowKeys;
     }
 
     public List<SmartShard> getRowKeysWithShardEnd(){
 
+        if(logger.isTraceEnabled()) {
+            logger.trace("Shards: {}", shards);
+        }
 
         final List<SmartShard> rowKeysWithShardEnd = new ArrayList<>(shards.size());
 
@@ -111,9 +121,14 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Colum
                 shardEnd = null;
             }
 
-            rowKeysWithShardEnd.add(new SmartShard(rowKey, shard.getShardIndex(), shardEnd));
+            rowKeysWithShardEnd.add(new SmartShard(rowKey, shard.getShardIndex(), shardEnd, shard.isDeleted()));
         }
 
+        if(logger.isTraceEnabled()) {
+            logger.trace("Resulting Smart Shards: {}", rowKeysWithShardEnd);
+        }
+
+
         return rowKeysWithShardEnd;
 
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index 47b630f..cf9a51c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Optional;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,7 +37,6 @@ import org.apache.usergrid.persistence.graph.SearchByEdgeType;
 import org.apache.usergrid.persistence.graph.exception.GraphRuntimeException;
 import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
 
-import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.inject.Inject;
 import com.netflix.astyanax.MutationBatch;
@@ -78,11 +78,10 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
 
 
     @Override
-    public Iterator<ShardEntryGroup> getShards( final ApplicationScope scope, final Optional<Shard> maxShardId,
-                                                final DirectedEdgeMeta directedEdgeMeta ) {
+    public Iterator<ShardEntryGroup> getShards(final ApplicationScope scope,
+                                               final DirectedEdgeMeta directedEdgeMeta) {
 
         ValidationUtils.validateApplicationScope( scope );
-        Preconditions.checkNotNull( maxShardId, "maxShardId cannot be null" );
         GraphValidation.validateDirectedEdgeMeta( directedEdgeMeta );
 
         Iterator<Shard> existingShards;
@@ -93,7 +92,7 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
         }
 
         else {
-            existingShards = edgeShardSerialization.getShardMetaData( scope, maxShardId, directedEdgeMeta );
+            existingShards = edgeShardSerialization.getShardMetaData( scope, Optional.absent(), directedEdgeMeta );
 
             /**
              * We didn't get anything out of cassandra, so we need to create the minimum shard

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
index 0a259f0..8e3be4f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
@@ -21,42 +21,26 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
 
 import java.beans.PropertyChangeEvent;
 import java.beans.PropertyChangeListener;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
+import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 
 import com.google.inject.Singleton;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
-import org.apache.usergrid.persistence.core.consistency.TimeService;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.util.ValidationUtils;
 import org.apache.usergrid.persistence.graph.GraphFig;
 import org.apache.usergrid.persistence.graph.exception.GraphRuntimeException;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardAllocation;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardCache;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.*;
 import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
 import org.apache.usergrid.persistence.graph.serialization.util.IterableUtil;
 
-import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import com.google.common.cache.Weigher;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListenableFutureTask;
 import com.google.common.util.concurrent.ListeningScheduledExecutorService;
@@ -164,7 +148,7 @@ public class NodeShardCacheImpl implements NodeShardCache {
 
         } else {
 
-            entry = new CacheEntry(nodeShardAllocation.getShards( key.scope, Optional.<Shard>absent(), key.directedEdgeMeta ));
+            entry = new CacheEntry(nodeShardAllocation.getShards( key.scope, key.directedEdgeMeta ));
 
         }
 
@@ -330,7 +314,7 @@ public class NodeShardCacheImpl implements NodeShardCache {
 
 
             final Iterator<ShardEntryGroup> edges =
-                nodeShardAllocation.getShards( key.scope, Optional.<Shard>absent(), key.directedEdgeMeta );
+                nodeShardAllocation.getShards( key.scope, key.directedEdgeMeta );
 
             final CacheEntry cacheEntry = new CacheEntry( edges );
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index 0853adb..8db491f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -346,7 +346,7 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
 
             //Overwrite our shard index with a newly created one that has been marked as compacted
             Shard compactedShard = new Shard( targetShard.getShardIndex(), timeService.getCurrentTime(), true );
-            compactedShard.setShardEnd(targetShard.getShardEnd());
+            compactedShard.setShardEnd(Optional.absent());
 
             if(logger.isTraceEnabled()) {
                 logger.trace("Shard has been fully compacted.  Marking shard {} as compacted in Cassandra", compactedShard);

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index 2fc0d50..7d91eda 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -191,21 +191,35 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
                 return DeleteResult.NO_OP;
             }
 
+            if(shard.isDeleted()){
+                if(logger.isTraceEnabled()){
+                    logger.trace("Shard {} already deleted.  Short circuiting.", shard);
+                }
+                return DeleteResult.NO_OP;
+            }
+
+            shard.setDeleted(true);
+
+            final MutationBatch setShardDeletedFlagMutation =
+                edgeShardSerialization.writeShardMeta(applicationScope, shard, directedEdgeMeta );
+
+            /* Previously the below was used for actually deleting the shard vs.a marking strategy with read filtering
 
-            final MutationBatch shardRemovalMutation =
-                edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
+                 final MutationBatch shardRemovalMutation =
+                    edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
+            */
 
             if ( rollup == null ) {
-                rollup = shardRemovalMutation;
+                rollup = setShardDeletedFlagMutation;
             }
 
             else {
-                rollup.mergeShallow( shardRemovalMutation );
+                rollup.mergeShallow( setShardDeletedFlagMutation );
             }
 
             result = DeleteResult.DELETED;
 
-            logger.info( "Removing shard {} in group {}", shard, shardEntryGroup );
+            logger.info( "{} - Marking shard {} as deleted in group {}", Thread.currentThread().getName()shard, shardEntryGroup );
         }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
index 65a6f40..55eb172 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
@@ -43,15 +43,7 @@ import org.apache.usergrid.persistence.graph.SearchByEdge;
 import org.apache.usergrid.persistence.graph.SearchByEdgeType;
 import org.apache.usergrid.persistence.graph.SearchByIdType;
 import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardStrategy;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerialization;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.*;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators.DescendingTimestampComparator;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators.OrderedComparator;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators
@@ -88,12 +80,15 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
     protected final GraphFig graphFig;
     protected final EdgeShardStrategy writeEdgeShardStrategy;
     protected final TimeService timeService;
+    protected final EdgeShardSerialization edgeShardSerialization;
+
 
 
     @Inject
     public ShardedEdgeSerializationImpl( final Keyspace keyspace, final CassandraConfig cassandraConfig,
                                          final GraphFig graphFig, final EdgeShardStrategy writeEdgeShardStrategy,
-                                         final TimeService timeService ) {
+                                         final TimeService timeService,
+                                         final EdgeShardSerialization edgeShardSerialization ) {
 
 
         checkNotNull( "keyspace required", keyspace );
@@ -101,6 +96,8 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         checkNotNull( "consistencyFig required", graphFig );
         checkNotNull( "writeEdgeShardStrategy required", writeEdgeShardStrategy );
         checkNotNull( "timeService required", timeService );
+        checkNotNull( "edgeShardSerialization required", edgeShardSerialization );
+
 
 
         this.keyspace = keyspace;
@@ -108,6 +105,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         this.graphFig = graphFig;
         this.writeEdgeShardStrategy = writeEdgeShardStrategy;
         this.timeService = timeService;
+        this.edgeShardSerialization = edgeShardSerialization;
     }
 
 
@@ -119,7 +117,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         GraphValidation.validateEdge( markedEdge );
         ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
 
-        return new SourceWriteOp( columnFamilies, markedEdge ) {
+        return new SourceWriteOp( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -144,7 +142,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
 
 
-        return new SourceTargetTypeWriteOp( columnFamilies, markedEdge ) {
+        return new SourceTargetTypeWriteOp( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -168,7 +166,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
 
 
-        return new TargetWriteOp( columnFamilies, markedEdge ) {
+        return new TargetWriteOp( columnFamilies, markedEdge, targetEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -194,7 +192,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
 
 
-        return new TargetSourceTypeWriteOp( columnFamilies, markedEdge ) {
+        return new TargetSourceTypeWriteOp( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -219,7 +217,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         ValidationUtils.verifyTimeUuid( timestamp, "timestamp" );
 
 
-        return new EdgeVersions( columnFamilies, markedEdge ) {
+        return new EdgeVersions( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -238,7 +236,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
                                                final MarkedEdge markedEdge, final Collection<Shard> shards,
                                                final DirectedEdgeMeta directedEdgeMeta, final UUID timestamp ) {
 
-        return new SourceWriteOp( columnFamilies, markedEdge ) {
+        return new SourceWriteOp( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -258,7 +256,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
                                                              final Collection<Shard> shards,
                                                              final DirectedEdgeMeta directedEdgeMeta,
                                                              final UUID timestamp ) {
-        return new SourceTargetTypeWriteOp( columnFamilies, markedEdge ) {
+        return new SourceTargetTypeWriteOp( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -279,7 +277,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
                                              final MarkedEdge markedEdge, final Collection<Shard> shards,
                                              final DirectedEdgeMeta directedEdgeMeta, final UUID timestamp ) {
 
-        return new TargetWriteOp( columnFamilies, markedEdge ) {
+        return new TargetWriteOp( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -300,7 +298,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
                                                            final DirectedEdgeMeta directedEdgeMeta,
                                                            final UUID timestamp ) {
 
-        return new TargetSourceTypeWriteOp( columnFamilies, markedEdge ) {
+        return new TargetSourceTypeWriteOp( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -320,7 +318,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
                                              final MarkedEdge markedEdge, final Collection<Shard> shards,
                                              final DirectedEdgeMeta directedEdgeMeta, final UUID timestamp ) {
 
-        return new EdgeVersions( columnFamilies, markedEdge ) {
+        return new EdgeVersions( columnFamilies, markedEdge, directedEdgeMeta ) {
 
             @Override
             void writeEdge( final MutationBatch batch,
@@ -696,6 +694,11 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
          */
         protected abstract boolean isDeleted();
 
+        /**
+         * Get the directed edge meta for the op
+         */
+        protected abstract DirectedEdgeMeta getDirectedEdgeMeta();
+
 
         /**
          * Write the edge with the given data
@@ -725,6 +728,16 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
             for ( Shard shard : shards ) {
                 final R rowKey = getRowKey( shard );
                 writeEdge( batch, columnFamily, scope, rowKey, column, shard, isDeleted );
+
+                // if an edge is being written to this shard, un-delete it in case it was previously marked
+                // don't un-delete if the edge write is to actually remove an edge
+                // Usergrid allows entities to be written with a UUID generated from the past (time)
+                if(shard.isDeleted() && !isDeleted) {
+                    logger.info("Shard is deleted. Un-deleting as new data is being written to the shard - {}", shard);
+                    shard.setDeleted(false);
+                    batch.mergeShallow(edgeShardSerialization.writeShardMeta(scope, shard, getDirectedEdgeMeta()));
+                }
+
             }
 
 
@@ -745,12 +758,14 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         private final String type;
         private final boolean isDeleted;
         private final DirectedEdge directedEdge;
+        private final DirectedEdgeMeta directedEdgeMeta;
 
 
         /**
          * Write the source write operation
          */
-        private SourceWriteOp( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge ) {
+        private SourceWriteOp( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge,
+                               final DirectedEdgeMeta directedEdgeMeta ) {
             this.columnFamily = edgeColumnFamilies.getSourceNodeCfName();
 
             this.sourceNodeId = markedEdge.getSourceNode();
@@ -759,6 +774,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
             this.isDeleted = markedEdge.isDeleted();
 
             this.directedEdge = new DirectedEdge( markedEdge.getTargetNode(), markedEdge.getTimestamp() );
+            this.directedEdgeMeta = directedEdgeMeta;
         }
 
 
@@ -784,6 +800,11 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         protected boolean isDeleted() {
             return isDeleted;
         }
+
+        @Override
+        protected DirectedEdgeMeta getDirectedEdgeMeta() {
+            return directedEdgeMeta;
+        }
     }
 
 
@@ -798,12 +819,14 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         private Id targetId;
         private final boolean isDeleted;
         private final DirectedEdge directedEdge;
+        private final DirectedEdgeMeta directedEdgeMeta;
 
 
         /**
          * Write the source write operation
          */
-        private SourceTargetTypeWriteOp( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge ) {
+        private SourceTargetTypeWriteOp( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge,
+                                         final DirectedEdgeMeta directedEdgeMeta ) {
             this.columnFamily = edgeColumnFamilies.getSourceNodeTargetTypeCfName();
 
             this.sourceNodeId = markedEdge.getSourceNode();
@@ -813,6 +836,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
             this.isDeleted = markedEdge.isDeleted();
 
             this.directedEdge = new DirectedEdge( targetId, markedEdge.getTimestamp() );
+            this.directedEdgeMeta = directedEdgeMeta;
         }
 
 
@@ -838,6 +862,11 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         protected boolean isDeleted() {
             return isDeleted;
         }
+
+        @Override
+        protected DirectedEdgeMeta getDirectedEdgeMeta() {
+            return directedEdgeMeta;
+        }
     }
 
 
@@ -853,12 +882,14 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         private final String type;
         private final boolean isDeleted;
         private final DirectedEdge directedEdge;
+        private final DirectedEdgeMeta directedEdgeMeta;
 
 
         /**
          * Write the source write operation
          */
-        private TargetWriteOp( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge ) {
+        private TargetWriteOp( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge,
+                               final DirectedEdgeMeta directedEdgeMeta ) {
             this.columnFamily = edgeColumnFamilies.getTargetNodeCfName();
 
             this.targetNode = markedEdge.getTargetNode();
@@ -867,6 +898,8 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
             this.isDeleted = markedEdge.isDeleted();
 
             this.directedEdge = new DirectedEdge( markedEdge.getSourceNode(), markedEdge.getTimestamp() );
+
+            this.directedEdgeMeta = directedEdgeMeta;
         }
 
 
@@ -892,6 +925,11 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         protected boolean isDeleted() {
             return isDeleted;
         }
+
+        @Override
+        protected DirectedEdgeMeta getDirectedEdgeMeta() {
+            return directedEdgeMeta;
+        }
     }
 
 
@@ -909,12 +947,14 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
 
         final boolean isDeleted;
         final DirectedEdge directedEdge;
+        final DirectedEdgeMeta directedEdgeMeta;
 
 
         /**
          * Write the source write operation
          */
-        private TargetSourceTypeWriteOp( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge ) {
+        private TargetSourceTypeWriteOp( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge,
+                                         final DirectedEdgeMeta directedEdgeMeta ) {
             this.columnFamily = edgeColumnFamilies.getSourceNodeTargetTypeCfName();
 
             this.targetNode = markedEdge.getTargetNode();
@@ -924,6 +964,8 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
             this.isDeleted = markedEdge.isDeleted();
 
             this.directedEdge = new DirectedEdge( sourceNode, markedEdge.getTimestamp() );
+
+            this.directedEdgeMeta = directedEdgeMeta;
         }
 
 
@@ -949,6 +991,11 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         protected boolean isDeleted() {
             return isDeleted;
         }
+
+        @Override
+        protected DirectedEdgeMeta getDirectedEdgeMeta() {
+            return directedEdgeMeta;
+        }
     }
 
 
@@ -967,11 +1014,14 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         final boolean isDeleted;
         final Long edgeVersion;
 
+        final DirectedEdgeMeta directedEdgeMeta;
+
 
         /**
          * Write the source write operation
          */
-        private EdgeVersions( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge ) {
+        private EdgeVersions( final EdgeColumnFamilies edgeColumnFamilies, final MarkedEdge markedEdge,
+                              final DirectedEdgeMeta directedEdgeMeta ) {
             this.columnFamily = edgeColumnFamilies.getGraphEdgeVersions();
 
             this.targetNode = markedEdge.getTargetNode();
@@ -981,6 +1031,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
             this.isDeleted = markedEdge.isDeleted();
 
             this.edgeVersion = markedEdge.getTimestamp();
+            this.directedEdgeMeta = directedEdgeMeta;
         }
 
 
@@ -1006,6 +1057,11 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
         protected boolean isDeleted() {
             return isDeleted;
         }
+
+        @Override
+        protected DirectedEdgeMeta getDirectedEdgeMeta() {
+            return directedEdgeMeta;
+        }
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/ShardSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/ShardSerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/ShardSerializer.java
index 8ab6288..b47f7cc 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/ShardSerializer.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/ShardSerializer.java
@@ -58,6 +58,7 @@ public class ShardSerializer extends AbstractSerializer<Shard> {
         }
 
         composite.addComponent( shard.isCompacted(), BOOLEAN_SERIALIZER);
+        composite.addComponent( shard.isDeleted(), BOOLEAN_SERIALIZER);
 
         return composite.serialize();
     }
@@ -67,7 +68,7 @@ public class ShardSerializer extends AbstractSerializer<Shard> {
     public Shard fromByteBuffer( final ByteBuffer byteBuffer ) {
 
         DynamicComposite composite = DynamicComposite.fromByteBuffer( byteBuffer );
-        Preconditions.checkArgument( composite.size() == 5, "Composite should 5 elements" );
+        Preconditions.checkArgument( composite.size() == 5 || composite.size() == 6, "Composite should 5 elements" );
 
 
         final byte version = composite.get(0, BYTE_SERIALIZER);
@@ -76,9 +77,15 @@ public class ShardSerializer extends AbstractSerializer<Shard> {
         final DirectedEdge shardEnd = composite.get( 3, EDGE_SERIALIZER);
         final boolean isCompacted = composite.get( 4, BOOLEAN_SERIALIZER);
 
-
         final Shard shard = new Shard(shardIndex, shardCreated, isCompacted);
         shard.setShardEnd(Optional.fromNullable(shardEnd));
+
+        if( composite.size() == 6){
+            final boolean isDeleted = composite.get( 5, BOOLEAN_SERIALIZER);
+            shard.setDeleted(isDeleted);
+        }
+
+
         return shard;
 
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index 0d6a27e..6e6abd8 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -22,12 +22,7 @@ package org.apache.usergrid.persistence.graph;
 import java.io.ByteArrayOutputStream;
 import java.io.PrintWriter;
 import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
@@ -136,7 +131,7 @@ public class GraphManagerShardConsistencyIT {
         // get the system property of the UUID to use.  If one is not set, use the defualt
         String uuidString = System.getProperty( "org.id", "80a42760-b699-11e3-a5e2-0800200c9a66" );
 
-        scope = new ApplicationScopeImpl( IdGenerator.createId( UUID.fromString( uuidString ), "test" ) );
+        scope = new ApplicationScopeImpl( IdGenerator.createId(UUIDGenerator.newTimeUUID(), "test" ) );
 
 
         reporter =
@@ -149,19 +144,24 @@ public class GraphManagerShardConsistencyIT {
 
 
     @After
-    public void tearDown() {
+    public void tearDown() throws Exception {
         reporter.stop();
         reporter.report();
 
-        if(writeExecutor != null){
+        if(writeExecutor != null && !writeExecutor.isShutdown()){
             writeExecutor.shutdownNow();
+            writeExecutor.awaitTermination(5000, TimeUnit.MILLISECONDS);
 
         }
-        if(deleteExecutor != null){
+        if(deleteExecutor != null && !deleteExecutor.isShutdown()){
             deleteExecutor.shutdownNow();
+            deleteExecutor.awaitTermination(5000, TimeUnit.MILLISECONDS);
+
 
         }
 
+        Thread.sleep(3000);
+
     }
 
 
@@ -251,7 +251,7 @@ public class GraphManagerShardConsistencyIT {
 
             for ( int i = 0; i < numWorkersPerInjector; i++ ) {
                 Future<Boolean> future =
-                    writeExecutor.submit( new Worker( gmf, generator, workerWriteLimit, minExecutionTime, writeCounter ) );
+                    writeExecutor.submit( new Worker( gmf, generator, workerWriteLimit, minExecutionTime, writeCounter) );
 
                 futures.add( future );
             }
@@ -299,7 +299,9 @@ public class GraphManagerShardConsistencyIT {
                 @Override
                 public void onSuccess( @Nullable final Long result ) {
                     logger.info( "Successfully ran the read, re-running" );
-                    writeExecutor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
+                    if( !writeExecutor.isShutdown() ) {
+                        writeExecutor.submit(new ReadWorker(gmf, generator, writeCount, readMeter));
+                    }
                 }
 
 
@@ -383,11 +385,12 @@ public class GraphManagerShardConsistencyIT {
         }
 
 
-        //now continue reading everything for 30 seconds
+        //now continue reading everything for 30 seconds to make sure things are OK
 
         Thread.sleep(30000);
 
-        writeExecutor.shutdownNow();
+        //writeExecutor.shutdownNow();
+        //writeExecutor.awaitTermination(5000, TimeUnit.MILLISECONDS);
     }
 
 
@@ -494,7 +497,7 @@ public class GraphManagerShardConsistencyIT {
 
             for ( int i = 0; i < numWorkersPerInjector; i++ ) {
                 Future<Boolean> future =
-                    deleteExecutor.submit( new Worker( gmf, generator, workerWriteLimit, minExecutionTime, writeCounter ) );
+                    deleteExecutor.submit( new Worker( gmf, generator, workerWriteLimit, minExecutionTime, writeCounter) );
 
                 futures.add( future );
             }
@@ -563,14 +566,18 @@ public class GraphManagerShardConsistencyIT {
                 .filter(markedEdge -> {
 
                     // if it's already been marked let's filter, move on as async deleteEdge()
-                    logger.trace("edge already marked, may indicated a problem with gm.deleteEdge(): {}", markedEdge);
-                    return !markedEdge.isDeleted();
+                    if(markedEdge.isDeleted()) {
+                        logger.info("Edge already marked, but gm.deleteEdge() is async, Edge: {}", markedEdge);
+                    }
+                    //return !markedEdge.isDeleted();
+                    return true;
                 })
                 .flatMap( edge -> manager.markEdge( edge ))
                      .flatMap( edge -> manager.deleteEdge( edge ) ).countLong().toBlocking().last();
 
             totalDeleted += count;
-            Thread.sleep( 500 );
+            logger.info("Sleeping 250ms second because deleteEdge() is async.");
+            Thread.sleep( 250 );
         }
 
 
@@ -591,7 +598,9 @@ public class GraphManagerShardConsistencyIT {
             @Override
             public void onSuccess( @Nullable final Long result ) {
                 logger.info( "Successfully ran the read, re-running" );
-                deleteExecutor.submit( new ReadWorker( gmf, generator, 0, readMeter ) );
+                if( !deleteExecutor.isShutdown() ) {
+                    deleteExecutor.submit(new ReadWorker(gmf, generator, 0, readMeter));
+                }
             }
 
 
@@ -641,7 +650,16 @@ public class GraphManagerShardConsistencyIT {
 
                 logger.info( "Shard size for group is {}", group.getReadShards() );
 
-                shardCount += group.getReadShards().size();
+                Collection<Shard> shards = group.getReadShards();
+
+                for(Shard shard: shards){
+
+                    if(!shard.isDeleted()){
+                        shardCount++;
+                    }
+                }
+
+                //shardCount += group.getReadShards().size();
             }
 
 
@@ -657,12 +675,337 @@ public class GraphManagerShardConsistencyIT {
         }
 
         //now that we have finished deleting and shards are removed, shutdown
-        deleteExecutor.shutdownNow();
+        //deleteExecutor.shutdownNow();
+        //deleteExecutor.awaitTermination(5000, TimeUnit.MILLISECONDS);
 
-        Thread.sleep( 3000 ); // sleep before the next test
     }
 
 
+
+    @Test(timeout=300000) // this test is SLOW as deletes are intensive and shard cleanup is async
+    @Category(StressTest.class)
+    public void writeThousandsDeleteWriteAgain()
+        throws InterruptedException, ExecutionException, MigrationException, UnsupportedEncodingException {
+
+        ConfigurationManager.getConfigInstance().setProperty( GraphFig.SHARD_SIZE, 2000 );
+
+
+        final Id sourceId = IdGenerator.createId( "sourceDeleteRewrite" );
+        final String deleteEdgeType = "testDeleteRewrite";
+
+        final List<Edge> edges = new ArrayList<>();
+
+        final EdgeGenerator generator = new EdgeGenerator() {
+
+
+            @Override
+            public Edge newEdge() {
+                Edge edge = createEdge( sourceId, deleteEdgeType, IdGenerator.createId( "targetDeleteRewrite" ) );
+
+                edges.add(edge);
+
+                return edge;
+            }
+
+
+            @Override
+            public Observable<MarkedEdge> doSearch( final GraphManager manager ) {
+                return manager.loadEdgesFromSource(
+                    new SimpleSearchByEdgeType( sourceId, deleteEdgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+                        Optional.<Edge>absent(), false ) );
+            }
+        };
+
+
+        final int numInjectors = 3;
+
+        /**
+         * create injectors.  This way all the caches are independent of one another.  This is the same as
+         * multiple nodes if there are multiple injectors
+         */
+        final List<Injector> injectors = createInjectors( numInjectors );
+
+
+        final GraphFig graphFig = getInstance( injectors, GraphFig.class );
+
+        final long shardSize = graphFig.getShardSize();
+
+
+        //we don't want to starve the cass runtime since it will be on the same box. Only take 50% of processing
+        // power for writes
+        final int numProcessors = Runtime.getRuntime().availableProcessors() / 2;
+
+        final int numWorkersPerInjector = numProcessors / numInjectors;
+
+        final long numberOfEdges = shardSize * TARGET_NUM_SHARDS;
+
+
+        final long workerWriteLimit = numberOfEdges / numWorkersPerInjector / numInjectors;
+
+        createDeleteExecutor( numWorkersPerInjector );
+
+
+        final AtomicLong writeCounter = new AtomicLong();
+
+
+        //min stop time the min delta + 1 cache cycle timeout
+        final long minExecutionTime = graphFig.getShardMinDelta() + graphFig.getShardCacheTimeout();
+
+
+        logger.info( "Writing {} edges per worker on {} workers in {} injectors", workerWriteLimit, numWorkersPerInjector,
+            numInjectors );
+
+
+        final List<Future<Boolean>> futures = new ArrayList<>();
+
+
+        for ( Injector injector : injectors ) {
+            final GraphManagerFactory gmf = injector.getInstance( GraphManagerFactory.class );
+
+
+            for ( int i = 0; i < numWorkersPerInjector; i++ ) {
+                Future<Boolean> future =
+                    deleteExecutor.submit( new Worker( gmf, generator, workerWriteLimit, minExecutionTime, writeCounter) );
+
+                futures.add( future );
+            }
+        }
+
+        /**
+         * Wait for all writes to complete
+         */
+        for ( Future<Boolean> future : futures ) {
+            future.get();
+        }
+
+        // now get all our shards
+        final NodeShardCache cache = getInstance( injectors, NodeShardCache.class );
+
+        final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( sourceId, deleteEdgeType );
+
+        final GraphManagerFactory gmf = getInstance( injectors, GraphManagerFactory.class );
+
+
+        final long writeCount = writeCounter.get();
+        final Meter readMeter = registry.meter( "readThroughput-deleteTestRewrite" );
+
+
+        //check our shard state
+
+
+        final Iterator<ShardEntryGroup> existingShardGroups =
+            cache.getReadShardGroup( scope, Long.MAX_VALUE, directedEdgeMeta );
+        int shardCount = 0;
+
+        while ( existingShardGroups.hasNext() ) {
+            final ShardEntryGroup group = existingShardGroups.next();
+
+            shardCount++;
+
+            logger.info( "Compaction pending status for group {} is {}", group, group.isCompactionPending() );
+        }
+
+
+        logger.info( "Found {} shard groups", shardCount );
+
+
+        //now mark and delete all the edges
+
+
+        final GraphManager manager = gmf.createEdgeManager( scope );
+
+        //sleep occasionally to stop pushing cassandra over
+
+        long count = Long.MAX_VALUE;
+
+        Thread.sleep(3000); // let's make sure everything is written
+
+        long totalDeleted = 0;
+
+        // now do the deletes
+        while(count != 0) {
+
+            logger.info("total deleted: {}", totalDeleted);
+            if(count != Long.MAX_VALUE) { // count starts with Long.MAX
+                logger.info("deleted {} entities, continuing until count is 0", count);
+            }
+            //take 1000 then sleep
+            count = generator.doSearch( manager ).take( 1000 )
+                .filter(markedEdge -> {
+
+                    // if it's already been marked let's filter, move on as async deleteEdge()
+                    if(markedEdge.isDeleted()){
+                        logger.info("Edge already marked, gm.deleteEdge() is Async, Edge: {}", markedEdge);
+                    }
+                    //return !markedEdge.isDeleted();
+                    return true;
+                })
+                .flatMap( edge -> manager.markEdge( edge ))
+                .flatMap( edge -> manager.deleteEdge( edge ) ).countLong().toBlocking().last();
+
+            totalDeleted += count;
+            logger.info("Sleeping 250ms second because deleteEdge() is an async process");
+            Thread.sleep( 250 );
+        }
+
+
+        logger.info("Sleeping before starting the read");
+        Thread.sleep(6000); //  let the edge readers start
+
+        // loop with a reader until our shards are gone
+
+
+        GraphManager gm = gmf.createEdgeManager( scope );
+
+
+
+
+        //do a read to eventually trigger our group compaction. Take 2 pages of columns
+        long returnedEdgeCount = generator.doSearch( gm )
+
+            .doOnNext( edge -> readMeter.mark() )
+
+            .countLong().toBlocking().last();
+
+        logger.info( "Completed reading {} edges", returnedEdgeCount );
+
+        if ( 0 != returnedEdgeCount ) {
+
+            //logger.warn( "Unexpected edge count returned!!!  Expected {} but was {}", 0,
+             //   returnedEdgeCount );
+
+            fail("Unexpected edge count returned!!!  Expected 0 but was "+ returnedEdgeCount );
+        }
+
+        logger.info("Got expected read count of 0");
+
+
+        //we have to get it from the cache, because this will trigger the compaction process
+        final Iterator<ShardEntryGroup> groups = cache.getReadShardGroup( scope, Long.MAX_VALUE, directedEdgeMeta );
+
+        ShardEntryGroup group = null;
+
+        while ( groups.hasNext() ) {
+
+            group = groups.next();
+            logger.info( "Shard size for group is {}", group.getReadShards() );
+            Collection<Shard> shards = group.getReadShards();
+
+            for(Shard shard: shards){
+                if(!shard.isDeleted()){
+                    shardCount++;
+                }
+            }
+        }
+
+
+        // we're done, 1 shard remains, we have a group, and it's our default shard
+        if ( shardCount == 1 && group.getMinShard().getShardIndex() == Shard.MIN_SHARD.getShardIndex()  ) {
+            logger.info( "All compactions complete," );
+
+        }
+
+
+        Thread.sleep( 2000 );
+
+
+        logger.info( "Re-Writing same edges", workerWriteLimit, numWorkersPerInjector,
+            numInjectors );
+
+
+        int edgeCount = 0;
+        if ( edges.size() > 0) {
+
+            for (Edge edge : edges) {
+
+                Edge returned = gm.writeEdge(edge).toBlocking().last();
+
+                assertNotNull("Returned has a version", returned.getTimestamp());
+
+                edgeCount++;
+
+                writeMeter.mark();
+
+                writeCounter.incrementAndGet();
+
+
+                if (edgeCount % 100 == 0) {
+                    logger.info("wrote: " + edgeCount);
+                }
+
+
+            }
+            logger.info("Re-wrote total: {}", edgeCount);
+        }
+
+        int retries = 2;
+        while ( retries > 0 ) {
+
+
+            //do a read to eventually trigger our group compaction. Take 2 pages of columns
+            returnedEdgeCount = generator.doSearch( gm )
+
+                .doOnNext( edge -> readMeter.mark() )
+
+                .countLong().toBlocking().last();
+
+            logger.info( "Completed reading {} edges", returnedEdgeCount );
+
+            if ( edgeCount != returnedEdgeCount ) {
+                logger.warn( "Unexpected edge count returned!!!  Expected {} but was {}", edgeCount,
+                    returnedEdgeCount );
+            }
+
+            retries--;
+
+            if( returnedEdgeCount == edgeCount ){
+                logger.info("Got expected read count of {}", edgeCount);
+                break;
+
+            }
+        }
+
+        //we have to get it from the cache, because this will trigger the compaction process
+        final Iterator<ShardEntryGroup> finalgroups = cache.getReadShardGroup( scope, Long.MAX_VALUE, directedEdgeMeta );
+
+        ShardEntryGroup finalgroup = null;
+
+        // reset the shard count
+        shardCount = 0;
+        while ( finalgroups.hasNext() ) {
+
+            finalgroup = finalgroups.next();
+
+            logger.info( "Shard size for group is {}", finalgroup.getReadShards() );
+
+            Collection<Shard> shards = finalgroup.getReadShards();
+
+            for(Shard shard: shards){
+
+                if(!shard.isDeleted()){
+                    shardCount++;
+                }
+            }
+
+        }
+
+
+        // we're done, 1 shard remains, we have a group, and it's our default shard
+        if ( shardCount > 1 ) {
+            logger.info( "All done. Final shard count: {}", shardCount );
+            assertEquals(1,1);
+
+        }else{
+            fail("There should be more than 1 shard");
+        }
+
+        //deleteExecutor.shutdownNow();
+        //deleteExecutor.awaitTermination(5000, TimeUnit.MILLISECONDS);
+
+    }
+
+
+
     private class Worker implements Callable<Boolean> {
         private final GraphManagerFactory factory;
         private final EdgeGenerator generator;
@@ -671,8 +1014,8 @@ public class GraphManagerShardConsistencyIT {
         private final AtomicLong writeCounter;
 
 
-        private Worker( final GraphManagerFactory factory, final EdgeGenerator generator, final long writeLimit,
-                        final long minExecutionTime, final AtomicLong writeCounter ) {
+        private Worker(final GraphManagerFactory factory, final EdgeGenerator generator, final long writeLimit,
+                       final long minExecutionTime, final AtomicLong writeCounter) {
             this.factory = factory;
             this.generator = generator;
             this.writeLimit = writeLimit;
@@ -699,7 +1042,6 @@ public class GraphManagerShardConsistencyIT {
                 assertNotNull( "Returned has a version", returned.getTimestamp() );
 
 
-                writeMeter.mark();
 
                 writeCounter.incrementAndGet();
 
@@ -738,7 +1080,7 @@ public class GraphManagerShardConsistencyIT {
             GraphManager gm = factory.createEdgeManager( scope );
 
 
-            while ( true ) {
+            while ( !Thread.currentThread().isInterrupted() ) {
 
 
                 //do a read to eventually trigger our group compaction. Take 2 pages of columns
@@ -757,6 +1099,8 @@ public class GraphManagerShardConsistencyIT {
 
                 assertEquals( "Expected to read same edge count", writeCount, returnedEdgeCount );
             }
+
+            return 0L;
         }
     }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
index 00406c0..8a9dcfe 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
@@ -564,7 +564,7 @@ public class NodeShardAllocationTest {
 
 
         final Iterator<ShardEntryGroup> result =
-                approximation.getShards( scope, Optional.<Shard>absent(), directedEdgeMeta );
+                approximation.getShards( scope, directedEdgeMeta );
 
 
         assertTrue( "Shards present", result.hasNext() );
@@ -663,7 +663,7 @@ public class NodeShardAllocationTest {
 
 
         final Iterator<ShardEntryGroup> result =
-                approximation.getShards( scope, Optional.<Shard>absent(), directedEdgeMeta );
+                approximation.getShards( scope, directedEdgeMeta );
 
 
         ShardEntryGroup shardEntryGroup = result.next();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/42c5c4b8/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java
index 3b717a4..ecf0091 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardCacheTest.java
@@ -102,7 +102,7 @@ public class NodeShardCacheTest {
         /**
          * Simulate returning no shards at all.
          */
-        when( allocation.getShards( same( scope ), same( max ), same( directedEdgeMeta ) ) )
+        when( allocation.getShards( same( scope ), same( directedEdgeMeta ) ) )
 
                 //use "thenAnswer" so we always return the value, even if  it's invoked more than 1 time.
                 .thenAnswer( new Answer<Iterator<ShardEntryGroup>>() {
@@ -191,7 +191,7 @@ public class NodeShardCacheTest {
         /**
          * Simulate returning no shards at all.
          */
-        when( allocation.getShards( same( scope ), any( Optional.class ), same( directedEdgeMeta ) ) )
+        when( allocation.getShards( same( scope ), same( directedEdgeMeta ) ) )
 
                 //use "thenAnswer" so we always return the value, even if  it's invoked more than 1 time.
                 .thenAnswer( new Answer<Iterator<ShardEntryGroup>>() {


[15/35] usergrid git commit: Make sure we skip over deleted shards if there are multiple together.

Posted by mr...@apache.org.
Make sure we skip over deleted shards if there are multiple together.


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1e6fb1e1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1e6fb1e1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1e6fb1e1

Branch: refs/heads/master
Commit: 1e6fb1e10d66c24d2c4b0d0f61a3312077545c97
Parents: 4440424
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Aug 29 10:11:19 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Aug 29 10:11:19 2016 -0700

----------------------------------------------------------------------
 .../core/astyanax/MultiRowShardColumnIterator.java          | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/1e6fb1e1/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
index 04266e1..2446968 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowShardColumnIterator.java
@@ -128,17 +128,20 @@ public class MultiRowShardColumnIterator<R, C, T> implements Iterator<T> {
             currentShard = currentShardIterator.next();
 
             // handle marked deleted shards
-            if( currentShard.isDeleted() && currentShardIterator.hasNext()){
+            while ( currentShard.isDeleted() && currentShardIterator.hasNext()){
 
                 if(logger.isTraceEnabled()) {
                     logger.trace("Shard is marked deleted, advancing to next - {}", currentShard);
                 }
 
                 currentShard = currentShardIterator.next();
-            }else if ( currentShard.isDeleted() && !currentShardIterator.hasNext()){
+            }
+
+            // if the last shard is deleted, return false, there is no next to seek
+            if ( currentShard.isDeleted() && !currentShardIterator.hasNext()){
 
                 if(logger.isTraceEnabled()) {
-                    logger.trace("Shard is marked deleted, and there is no more - {}", currentShard);
+                    logger.trace("Shard is marked deleted, and there are no more shards - {}", currentShard);
                 }
 
                 return false;