You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2016/12/13 15:43:19 UTC
[1/3] usergrid git commit: Add option to update each entity.
Repository: usergrid
Updated Branches:
refs/heads/master 844468725 -> 79492c294
Add option to update each entity.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/33fe480a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/33fe480a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/33fe480a
Branch: refs/heads/master
Commit: 33fe480a0c0585ac0a4861eda996cce6e84cce94
Parents: 8444687
Author: Dave Johnson <sn...@gmail.com>
Authored: Tue Dec 13 09:11:49 2016 -0500
Committer: Dave Johnson <sn...@gmail.com>
Committed: Tue Dec 13 09:11:49 2016 -0500
----------------------------------------------------------------------
stack/tools/pom.xml | 10 ++++-
.../org/apache/usergrid/tools/Repersist.java | 46 ++++++++++++++------
.../apache/usergrid/tools/RepersistTest.java | 6 +++
3 files changed, 48 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/33fe480a/stack/tools/pom.xml
----------------------------------------------------------------------
diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml
index 3270683..817cbb6 100644
--- a/stack/tools/pom.xml
+++ b/stack/tools/pom.xml
@@ -54,14 +54,15 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
+
<configuration>
- <skipTests>false</skipTests>
<systemPropertyVariables>
<storage-config>${basedir}/src/test/conf</storage-config>
</systemPropertyVariables>
<forkMode>always</forkMode>
<argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
</configuration>
+
</plugin>
<plugin>
@@ -260,5 +261,12 @@
<version>${aws.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/33fe480a/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java
index f16b2dc..9d023e3 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Repersist.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.tools;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
@@ -31,17 +32,30 @@ import java.util.UUID;
/**
- * Iterate through all data in a list of apps and re-persist every entity in every collection.
+ * Iterate through all data in a list of apps and get each or update entity.
*/
public class Repersist extends ToolBase {
static final Logger logger = LoggerFactory.getLogger( Repersist.class );
+ int count = 0;
+
@Override
@SuppressWarnings("static-access")
public Options createOptions() {
+
Options options = super.createOptions();
- options.addOption("apps", true, "Comma-separated list of apps to re-persisted in {appName}/{orgName} format");
- options.addOption("wait (ms)", true, "Time to wait between entity re-persist");
+
+ Option appsOpton = new Option("apps", true,
+ "Comma-separated list of apps to re-persisted in {appName}/{orgName} format");
+ appsOpton.setRequired(true);
+ options.addOption( appsOpton );
+
+ options.addOption("wait", true,
+ "Time in milliseconds to wait between entity re-persist");
+
+ options.addOption("update", false,
+ "Tool will update each and every entity in the listed apps");
+
return options;
}
@@ -55,23 +69,26 @@ public class Repersist extends ToolBase {
long wait = 100;
try {
- Long.parseLong(line.getOptionValue("wait"));
+ wait = Long.parseLong(line.getOptionValue("wait"));
} catch( Exception e ) {
logger.error("Incorrect or missing wait time, using default: " + wait);
}
+ boolean update = line.hasOption("update");
+
for ( String orgApp : orgApps ) {
String[] orgAppParts = orgApp.split("/");
String org = orgAppParts[0];
String app = orgAppParts[1];
- repersist( org, org + "/" + app, wait );
+ repersist( org, org + "/" + app, update, wait );
}
}
- private void repersist( String organizationName, String applicationName, long wait ) throws Exception {
+ private void repersist( String organizationName, String applicationName, boolean update, long wait )
+ throws Exception {
logger.info( "\n\nRepersisting {}/{}\n", organizationName, applicationName );
@@ -87,25 +104,29 @@ public class Repersist extends ToolBase {
for ( String collectionName : collectionMetadata.keySet() ) {
- int count = 0;
-
Query query = new Query();
query.setLimit( MAX_ENTITY_FETCH );
Results results = em.searchCollection( em.getApplicationRef(), collectionName, query );
while (results.size() > 0) {
- for (Entity entity : results.getEntities()) {
- // repersist
- em.update(entity);
+ for (Entity entity : results.getEntities()) {
+ if ( update ) {
+ // TODO: can we do this without updating the update of each entity?
+ em.update(entity);
+ }
count++;
+ if ( count % 1000 == 0 ) {
+ logger.info("Processed {} entities", count);
+ }
Thread.sleep( wait );
-
}
+
if (results.getCursor() == null) {
break;
}
+
query.setCursor( results.getCursor() );
results = em.searchCollection( em.getApplicationRef(), collectionName, query );
}
@@ -114,7 +135,6 @@ public class Repersist extends ToolBase {
organizationName, applicationName, collectionName, count);
}
-
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/33fe480a/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java
index 4d65c16..348799b 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/RepersistTest.java
@@ -24,6 +24,8 @@ import org.junit.ClassRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.junit.Assert.assertEquals;
+
public class RepersistTest extends AbstractServiceIT {
static final Logger logger = LoggerFactory.getLogger( RepersistTest.class );
@@ -48,6 +50,8 @@ public class RepersistTest extends AbstractServiceIT {
"-organization", orgName,
"-application", appName,
"-username", userName,
+ "-users", "5",
+ "-collections", "1",
"-host", "localhost:9160",
"-eshost", "localhost:9200",
"-escluster", "elasticsearch",
@@ -67,6 +71,8 @@ public class RepersistTest extends AbstractServiceIT {
}, false);
+ assertEquals( 38, repersist.count );
+
}
}
[3/3] usergrid git commit: Fix ExportAppTest
Posted by sn...@apache.org.
Fix ExportAppTest
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/79492c29
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/79492c29
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/79492c29
Branch: refs/heads/master
Commit: 79492c294c3a4ec25c60319fd6512136351ef958
Parents: cb95ce5
Author: Dave Johnson <sn...@gmail.com>
Authored: Tue Dec 13 10:42:55 2016 -0500
Committer: Dave Johnson <sn...@gmail.com>
Committed: Tue Dec 13 10:42:55 2016 -0500
----------------------------------------------------------------------
.../src/test/java/org/apache/usergrid/tools/ExportAppTest.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/79492c29/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
index b7bb2cc..ae32cc8 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
@@ -62,6 +62,8 @@ public class ExportAppTest extends AbstractServiceIT {
"-organization", orgName,
"-application", appName,
"-username", userName,
+ "-users", "10",
+ "-collections", "5",
"-host", "localhost:9160",
"-eshost", "localhost:9200",
"-escluster", "elasticsearch",
@@ -103,7 +105,8 @@ public class ExportAppTest extends AbstractServiceIT {
"-host", "localhost:9160",
"-eshost", "localhost:9200",
"-escluster", "elasticsearch",
- "-outputDir", directoryName + "1"
+ "-outputDir", directoryName + "1",
+ "-ugcluster","usergrid"
}, false );
logger.info( "1 thread time = " + (System.currentTimeMillis() - start) / 1000 + "s" );
[2/3] usergrid git commit: Fix the rarely used getConnectionTypes()
method to use Graph rather than Dictionary.
Posted by sn...@apache.org.
Fix the rarely used getConnectionTypes() method to use Graph rather than Dictionary.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cb95ce5a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cb95ce5a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cb95ce5a
Branch: refs/heads/master
Commit: cb95ce5afcc2dbfe1ac74ab3e6ebdd8d93e1207d
Parents: 33fe480
Author: Dave Johnson <sn...@gmail.com>
Authored: Tue Dec 13 09:49:41 2016 -0500
Committer: Dave Johnson <sn...@gmail.com>
Committed: Tue Dec 13 09:49:41 2016 -0500
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 18 ++++++++++--------
.../corepersistence/util/CpNamingUtils.java | 1 -
.../usergrid/persistence/EntityConnectionsIT.java | 4 ++++
3 files changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cb95ce5a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 65806af..4a759ac 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -832,15 +832,17 @@ public class CpRelationManager implements RelationManager {
@Override
public Set<String> getConnectionTypes( boolean filterConnection ) throws Exception {
- Set<String> connections = cast( em.getDictionaryAsSet( headEntity, Schema.DICTIONARY_CONNECTED_TYPES ) );
- if ( connections == null ) {
- return null;
- }
- if ( filterConnection && ( connections.size() > 0 ) ) {
- connections.remove( "connection" );
- }
- return connections;
+ final GraphManager gm = managerCache.getGraphManager( applicationScope );
+
+ Observable<String> edges =
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( cpHeadEntity.getId(), null, null ) );
+
+ return edges.collect( () -> new HashSet<String>(), ( edgeSet, edge ) -> {
+ edgeSet.add( CpNamingUtils.getNameFromEdgeType( edge ) );
+
+ } ).toBlocking().last();
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cb95ce5a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 3ebab39..9c6e318 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -321,7 +321,6 @@ public class CpNamingUtils {
*/
public static String getNameFromEdgeType( final String edgeName ) {
-
if ( isCollectionEdgeType( edgeName ) ) {
return getCollectionName( edgeName ) ;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cb95ce5a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
index d9ee960..296bf53 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
@@ -60,6 +60,10 @@ public class EntityConnectionsIT extends AbstractCoreIT {
em.createConnection( firstUserEntity, "likes", secondUserEntity );
+ Set<String> connectionTypes = em.getConnectionTypes(firstUserEntity);
+ assertEquals( 1, connectionTypes.size());
+ assertEquals("likes", connectionTypes.iterator().next());
+
app.refreshIndex();
Results r = em.getTargetEntities(firstUserEntity, "likes", null, Level.IDS);