You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/10/31 00:32:35 UTC
[01/50] [abbrv] usergrid git commit: Add endpoint for setting apigee
mobile config properties on the application.
Repository: usergrid
Updated Branches:
refs/heads/master 9d3cf5b8b -> b031d4d3d
Add endpoint for setting apigee mobile config properties on the application.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/88d11790
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/88d11790
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/88d11790
Branch: refs/heads/master
Commit: 88d11790c2d077140a71347f97ad5bc75346af2c
Parents: f3693c8
Author: Michael Russo <mi...@gmail.com>
Authored: Wed Oct 21 21:16:50 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Wed Oct 21 21:16:50 2015 -0700
----------------------------------------------------------------------
.../rest/applications/ApplicationResource.java | 50 ++++++++++++++++++++
1 file changed, 50 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/88d11790/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
index c521d4f..4eaf09a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.rest.applications;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.sun.jersey.api.json.JSONWithPadding;
import com.sun.jersey.api.view.Viewable;
import org.apache.amber.oauth2.common.error.OAuthError;
@@ -47,6 +48,7 @@ import org.apache.usergrid.rest.exceptions.NotFoundExceptionMapper;
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.oauth.ClientCredentialsInfo;
import org.slf4j.Logger;
@@ -601,4 +603,52 @@ public class ApplicationResource extends ServiceResource {
}
return new JSONWithPadding( value, callback );
}
+
+ // Specifically require superuser access as this is setting app properties directly (only way to currently do this
+ // with Apigee's apigeeMobileConfig
+ @RequireSystemAccess
+ @POST
+ @Path("apm/apigeeMobileConfig")
+ @Consumes(APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public JSONWithPadding setAPMConfig( @Context UriInfo ui,
+ @QueryParam("callback") @DefaultValue("callback") String callback,
+ Map<String, Object> json) throws Exception {
+
+ if(json == null || json.size() < 1){
+ logger.error("Param {} cannot be null for POST apm/apigeeMobileConfig", APIGEE_MOBILE_APM_CONFIG_JSON_KEY);
+ throw new IllegalArgumentException("Request body cannot be empty and must include apigeeMobileConfig params");
+ }
+
+ final String requestAppUUID = (String) json.get("applicationUUID");
+ if(!requestAppUUID.equalsIgnoreCase(applicationId.toString())){
+ logger.error("Provided application UUID {} does not match actual application UUID {}",
+ requestAppUUID,
+ applicationId.toString());
+ throw new IllegalArgumentException(
+ String.format("Provided application UUID %s does not match actual application UUID %s",
+ requestAppUUID,
+ applicationId.toString())
+ );
+ }
+
+ final String apmConfig = new ObjectMapper().writeValueAsString(json);
+ if(logger.isDebugEnabled()){
+ logger.debug("Received request to set apigeeMobileConfig properties with: {}", apmConfig);
+ }
+
+
+ EntityManager em = emf.getEntityManager( applicationId );
+ em.setProperty(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
+ APIGEE_MOBILE_APM_CONFIG_JSON_KEY,
+ apmConfig
+ );
+
+ logger.info("Successfully set apigeeMobileConfig properties with: {}", apmConfig);
+
+ return new JSONWithPadding(apmConfig, callback);
+
+ }
+
+
}
[39/50] [abbrv] usergrid git commit: Ensure that status is updated
properly.
Posted by to...@apache.org.
Ensure that status is updated properly.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f8c703c0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f8c703c0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f8c703c0
Branch: refs/heads/master
Commit: f8c703c02c1182ad63ad86587749eb1ae09c202a
Parents: 471dc35
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 17:57:32 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 17:57:32 2015 -0400
----------------------------------------------------------------------
.../rest/system/ConnectionResource.java | 48 +++++++++++++-------
1 file changed, 32 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f8c703c0/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
index 6e683ed..14b79f3 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
@@ -55,6 +55,7 @@ import com.google.common.base.Preconditions;
import com.sun.jersey.api.json.JSONWithPadding;
import rx.Observable;
+import rx.functions.Action1;
import rx.schedulers.Schedulers;
@@ -146,36 +147,51 @@ public class ConnectionResource extends AbstractContextResource {
//start de duping and run in the background
connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
- .doOnNext( buffer -> {
+ .doOnNext(buffer -> {
- final long runningTotal = count.addAndGet( buffer.size() );
+ final long runningTotal = count.addAndGet(buffer.size());
final Map<String, Object> status = new HashMap<String, Object>() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", System.currentTimeMillis() );
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", System.currentTimeMillis());
}};
- statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
- StatusService.Status.INPROGRESS, status ).toBlocking().lastOrDefault( null );
- } ).doOnSubscribe( () -> {
- statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.STARTED,
- new HashMap<>() ).toBlocking().lastOrDefault( null );
- } ).doOnCompleted( () -> {
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
+ StatusService.Status.INPROGRESS, status).toBlocking().lastOrDefault(null);
+ }).doOnSubscribe(() -> {
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.STARTED, new HashMap<>()).toBlocking().lastOrDefault(null);
+
+ }).doOnCompleted(() -> {
final long runningTotal = count.get();
final Map<String, Object> status = new HashMap<String, Object>() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", System.currentTimeMillis() );
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", System.currentTimeMillis());
}};
- statusService
- .setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.COMPLETE, status );
- } ).subscribeOn( Schedulers.newThread() ).subscribe();
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.COMPLETE, status).toBlocking().lastOrDefault(null);
+
+ }).doOnError( (throwable) -> {
+ logger.error("Error deduping connections", throwable);
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put("error", throwable.getMessage() );
+ }};
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.FAILED, status).toBlocking().lastOrDefault(null);;
+
+ } ).subscribeOn(Schedulers.newThread()).subscribe();
+
+ final StatusService.JobStatus status =
+ new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
- final StatusService.JobStatus status = new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
return createResult( status, callback );
}
[14/50] [abbrv] usergrid git commit: Merge branch 'USERGRID-1062' of
https://git-wip-us.apache.org/repos/asf/usergrid into USERGRID-1052
Posted by to...@apache.org.
Merge branch 'USERGRID-1062' of https://git-wip-us.apache.org/repos/asf/usergrid into USERGRID-1052
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a770a6d7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a770a6d7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a770a6d7
Branch: refs/heads/master
Commit: a770a6d7b99389031fe957372cb867d6a06dfcb9
Parents: 1d4f22c 032ffd4
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 23 15:48:17 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 23 15:48:17 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 7 +++---
.../asyncevents/EventBuilder.java | 25 +++++++++++++++-----
.../asyncevents/EventBuilderImpl.java | 9 +++----
.../asyncevents/InMemoryAsyncEventService.java | 5 ++--
4 files changed, 30 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
[32/50] [abbrv] usergrid git commit: First and untested stab at
multi-tenant migration.
Posted by to...@apache.org.
First and untested stab at multi-tenant migration.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d79bf4c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d79bf4c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d79bf4c3
Branch: refs/heads/master
Commit: d79bf4c36810fdab578a86e4de0ae725f1aa6e75
Parents: 2686054
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 08:56:14 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 08:56:14 2015 -0400
----------------------------------------------------------------------
stack/scripts/create_test_data.py | 213 +++++++++++++++++++++
stack/scripts/migrate_entity_data.py | 301 +++++++++++++++++++++---------
2 files changed, 429 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d79bf4c3/stack/scripts/create_test_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/create_test_data.py b/stack/scripts/create_test_data.py
new file mode 100644
index 0000000..de85da0
--- /dev/null
+++ b/stack/scripts/create_test_data.py
@@ -0,0 +1,213 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+import logging
+from logging.handlers import RotatingFileHandler
+import argparse
+import time
+import requests
+import json
+
+# Creates two organizations each with two apps each with three collections each with 100 entities
+# Caller must provide a "slug" string which will be used as a prefix for all names
+#
+# For example, if the slug is mytest then:
+# Orgs will be named mytest_org0 and mytest_org1
+# Apps will be named mytest_org0_app0 and so on
+# Collections will be named mytest_org0_app0_col0 and so on
+# Entities will be named mytest_org0_app0_col0_entity and so on
+# Org admins will be named mytest_org0_admin and mytest_org1_admin (both with password test)
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Usergrid Test Data Creation Tool")
+
+ parser.add_argument("--endpoint",
+ help="The endpoint to use for making API requests.",
+ type=str,
+ default="http://localhost:8080")
+
+ parser.add_argument("--user",
+ help="System Admin Credentials used to authenticate with Usergrid <user:pass>",
+ type=str,
+ required=True)
+
+ parser.add_argument("--slug",
+ help="Unique string to be used to name organization, applications and other things create",
+ type=str,
+ required=True)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ arg_vars = vars(my_args)
+ creds = arg_vars["user"].split(":")
+ if len(creds) != 2:
+ print("Credentials not properly specified. Must be '-u <user:pass>'. Exiting...")
+ exit_on_error()
+ else:
+ arg_vars["user"] = creds[0]
+ arg_vars["pass"] = creds[1]
+
+ return arg_vars
+
+
+class Creator:
+ def __init__(self):
+ self.args = parse_args()
+ self.endpoint = self.args["endpoint"]
+ self.logger = init_logging(self.__class__.__name__)
+ self.admin_user = self.args["user"]
+ self.admin_pass = self.args["pass"]
+ self.slug = self.args["slug"]
+
+ def run(self):
+ self.logger.info("Initializing...")
+
+ if not self.is_endpoint_available():
+ exit_on_error("Endpoint is not available, aborting")
+
+ for orgIndex in range(2):
+ orgName = self.slug + "_org" + str(orgIndex)
+ orgUser = orgName + "_admin"
+ orgEmail = orgUser + "@example.com"
+
+ url = self.endpoint + "/management/orgs"
+ body = json.dumps({"username":orgUser, "email":orgEmail, "password":"test", "organization":orgName })
+ r = requests.post(url=url, data=body, auth=(self.admin_user, self.admin_pass))
+ if ( r.status_code >= 400 ):
+ print "Error creating organization " + orgName + ": " + r.text
+ return
+
+ print "Created org " + orgName
+
+ url = self.endpoint + "/management/token"
+ body = json.dumps({"grant_type":"password","username":orgUser,"password":"test"})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code != 200 ):
+ print "Error logging into organization " + orgName + ": " + r.text
+ return
+
+ accessToken = r.json()["access_token"]
+
+ for appIndex in range(2):
+ appName = orgName + "_app" + str(appIndex)
+
+ url = self.endpoint + "/management/orgs/" + orgName + "/apps?access_token=" + accessToken
+ body = json.dumps({"name":appName})
+ r = requests.post(url=url, data=body, auth=(self.admin_user, self.admin_pass))
+ if ( r.status_code >= 400 ):
+ print "Error creating application" + appName + ": " + r.text
+ return
+
+ print " Created app: " + orgName + "/" + appName
+ appUrl = self.endpoint + "/" + orgName + "/" + appName
+ time.sleep(2)
+
+ for userIndex in range(2):
+ userName = appName + "_user" + str(userIndex)
+ email = userName + "@example.com"
+
+ url = appUrl + "/users?access_token=" + accessToken
+ body = json.dumps({"name":userName, "username":userName, "email":email, "password":"test"})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code >= 400 ):
+ print "Error creating user " + userName + ": " + r.text
+ return
+
+ for colIndex in range(3):
+ colName = appName + "_col" + str(colIndex)
+ print " Creating collection: " + colName
+
+ for entityIndex in range(100):
+ entityName = colName + "_entity" + str(entityIndex)
+
+ url = appUrl + "/" + colName + "s?access_token=" + accessToken
+ body = json.dumps({"name":entityName})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code >= 400 ):
+ print "Error creating entity" + userName + ": " + r.text
+ retur
+
+ # connect entities in collection 0 to collection 1
+ for entityIndex in range(100):
+ sourceCollection = appName + "_col0s"
+ sourceName = appName + "_col0_entity" + str(entityIndex)
+ targetName = appName + "_col1_entity" + str(entityIndex)
+ targetType = appName + "_col1"
+ url = appUrl + "/" + sourceCollection + "/" + sourceName + "/has/" + targetType + "/" + targetName
+ r = requests.post(url=url + "?access_token=" + accessToken)
+ if ( r.status_code >= 400 ):
+ print "Error connecting entity " + sourceName + " to " + targetName + ": " + r.text
+ print "url is: " + url
+ return
+
+ # connect entities in collection 1 to collection 2
+ for entityIndex in range(100):
+ sourceCollection = appName + "_col1s"
+ sourceName = appName + "_col1_entity" + str(entityIndex)
+ targetName = appName + "_col2_entity" + str(entityIndex)
+ targetType = appName + "_col2"
+ url = appUrl + "/" + sourceCollection + "/" + sourceName + "/has/" + targetType + "/" + targetName
+ r = requests.post(url=url + "?access_token=" + accessToken)
+ if ( r.status_code >= 400 ):
+ print "Error connecting entity " + sourceName + " to " + targetName + ": " + r.text
+ print "url is: " + url
+ return
+
+ def is_endpoint_available(self):
+
+ try:
+ r = requests.get(url=self.endpoint+"/status")
+ if r.status_code == 200:
+ return True
+ except requests.exceptions.RequestException as e:
+ self.logger.error("Endpoint is unavailable, %s", str(e))
+ return False
+
+
+def exit_on_error(e=""):
+ print ("Exiting script due to error: " + str(e))
+ sys.exit(1)
+
+
+def init_logging(name):
+
+ logger = logging.getLogger(name)
+ log_file_name = "./create-test-data.log"
+ log_formatter = logging.Formatter(fmt="%(asctime)s [%(name)s] %(levelname)s %(message)s",
+ datefmt="%Y-%m-%d %H:%M:%S")
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode="a",
+ maxBytes=104857600,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+ logger.addHandler(rotating_file)
+ logger.setLevel(logging.INFO)
+
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+ logger.addHandler(stdout_logger)
+
+ return logger
+
+if __name__ == "__main__":
+
+ creator = Creator()
+ creator.run()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d79bf4c3/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index 36f73c0..f576108 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -16,30 +16,50 @@
# under the License.
#
#
+# To migrate multiple tenants within one cluster.
#
-# Usage from a machine running Usergrid with the new Usergrid version:
+# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
#
-# ######################################################
-# STEP 1 - BEFORE SWITCHING TRAFFIC TO NEW UG VERSION
-# ######################################################
+# python migrate_entity_data.py --org <org1name> --user <superuser>:<superpass> --init
#
-# python migrate_entity_data.py --user adminuser:adminpass
+# This command will setup and bootstrap the database, setup the migration system and update index mappings:
+# - /system/database/setup
+# - /system/database/bootstrap
+# - /system/migrate/run/migration-system
+# - /system/migrate/run/index_mapping_migration
#
-# The above command performs an appinfo migration and system re-index only. This creates indices in Elasticsearch with
-# the updated indexing strategy in the new Usergrid version.
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app.
#
-# ######################################################
-# STEP 2 - AFTER SWITCHING TRAFFIC TO NEW UG VERSION
-# ######################################################
+# STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
#
-# python migrate_entity_data.py --user adminuser:adminpass --delta --date <timestamp>
+# python migrate_entity_data.py --org <org1name> --user <superuser>:<superpass> --date
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app with a start-date specified so only data modified since
+# STEP 1 will be re-indexed.
+#
+# STEP 3 - SETUP TENENT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+#
+# python migrate_entity_data.py --org <org2name> --user <superuser>:<superpass> --date
+#
+# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app.
+#
+# STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
+#
+# python migrate_entity_data.py --org <org2name> --user <superuser>:<superpass> --date
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app with a start-date specified so only data modified since
+# STEP 1 will be re-indexed.
+#
+# STEP 5 - FULL DATA MIGRATION
+#
+# python migrate_entity_data.py --user <superuser>:<superpass> --full
+#
+# This command will run the full data migration.
#
-# The above command performs an appinfo migration, system re-index using a start date, and full data migration which
-# includes entity data. This step is necessary to ensure Usergrid starts reading and writing data from the latest
-# entity version, including delta indexing of any documents create during the time between STEP 1 and STEP 2. If
-# all data has already been migrated (running this a 2nd, 3rd, etc. time), then the appinfo migration will be skipped.
-
-
import sys
import logging
@@ -72,10 +92,6 @@ PLUGIN_CORE_DATA = 'core-data'
def parse_args():
parser = argparse.ArgumentParser(description='Usergrid Migration Tool')
- parser.add_argument('--date',
- help='A date from which to start the migration',
- type=str)
-
parser.add_argument('--endpoint',
help='The endpoint to use for making API requests.',
type=str,
@@ -86,8 +102,22 @@ def parse_args():
type=str,
required=True)
- parser.add_argument('--delta',
- help='Run a delta migration.',
+ parser.add_argument('--init',
+ help='Init system and start first migration.',
+ action='store_true',
+ default=False)
+
+ parser.add_argument('--org',
+ help='Name of organization on which to run migration.',
+ type=str,
+ required=False)
+
+ parser.add_argument('--date',
+ help='A date from which to start the migration',
+ type=str)
+
+ parser.add_argument('--full',
+ help='Run full data migration (last step in cluster migration).',
action='store_true',
default=False)
@@ -120,7 +150,9 @@ class Migrate:
self.logger = init_logging(self.__class__.__name__)
self.admin_user = self.args['user']
self.admin_pass = self.args['pass']
- self.delta_migration = self.args['delta']
+ self.org = self.args['org']
+ self.init = self.args['init']
+ self.full = self.args['full']
def run(self):
self.logger.info('Initializing...')
@@ -133,62 +165,82 @@ class Migrate:
try:
- self.run_database_setup()
+ if self.full:
- # We need to check and roll the migration system to 1 if not already
- migration_system_updated = self.is_migration_system_updated()
+ # Do full data migration and exit
- if not migration_system_updated:
- self.logger.info('Migration system needs to be updated. Updating migration system..')
- self.start_migration_system_update()
- while not migration_system_updated:
+ self.start_fulldata_migration()
+
+ self.metrics['full_data_migration_start'] = get_current_time()
+ self.logger.info("Full Data Migration Started")
+ is_migrated = False
+ while not is_migrated:
time.sleep(STATUS_INTERVAL_SECONDS)
- migration_system_updated = self.is_migration_system_updated()
- if migration_system_updated:
+ is_migrated = self.is_data_migrated()
+ if is_migrated:
break
- index_mapping_updated = self.is_index_mapping_updated()
+ self.metrics['full_data_migration_end'] = get_current_time()
+ self.logger.info("Full Data Migration completed")
- if not index_mapping_updated:
- self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
- self.start_index_mapping_migration()
- while not index_mapping_updated:
- time.sleep(STATUS_INTERVAL_SECONDS)
- index_mapping_updated = self.is_index_mapping_updated()
- if index_mapping_updated:
- break
+ self.log_metrics()
+ self.logger.info("Finished...")
- # Run AppInfo migration only when both appinfos and collection entity data have not been migrated
- if not self.is_data_migrated():
+ return
- #Migrate app info
- if self.is_appinfo_migrated():
- self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
- self.reset_appinfo_migration()
- time.sleep(STATUS_INTERVAL_SECONDS)
- self.start_appinfo_migration()
- self.logger.info('AppInfo Migration Started.')
- self.metrics['appinfo_migration_start'] = get_current_time()
+ if self.init:
- is_appinfo_migrated = False
- while not is_appinfo_migrated:
- is_appinfo_migrated = self.is_appinfo_migrated()
- time.sleep(STATUS_INTERVAL_SECONDS)
- if is_appinfo_migrated:
- self.metrics['appinfo_migration_end'] = get_current_time()
- break
- self.logger.info('AppInfo Migration Ended.')
+ # Init the migration system as this is the first migration done on the cluster
+ self.run_database_setup()
- else:
- self.logger.info('Full Data Migration previously ran... skipping AppInfo migration.')
+ migration_system_updated = self.is_migration_system_updated()
+ if not migration_system_updated:
+ self.logger.info('Migration system needs to be updated. Updating migration system..')
+ self.start_migration_system_update()
+ while not migration_system_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ migration_system_updated = self.is_migration_system_updated()
+ if migration_system_updated:
+ break
+ index_mapping_updated = self.is_index_mapping_updated()
- # We need to check and roll index mapping version to 1 if not already there
+ if not index_mapping_updated:
+ self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
+ self.start_index_mapping_migration()
+ while not index_mapping_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ index_mapping_updated = self.is_index_mapping_updated()
+ if index_mapping_updated:
+ break
+
+
+ # Migrate app info
+
+ if self.is_appinfo_migrated():
+ self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
+ self.reset_appinfo_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+
+ self.start_appinfo_migration()
+ self.logger.info('AppInfo Migration Started.')
+ self.metrics['appinfo_migration_start'] = get_current_time()
+
+ is_appinfo_migrated = False
+ while not is_appinfo_migrated:
+ is_appinfo_migrated = self.is_appinfo_migrated()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ if is_appinfo_migrated:
+ self.metrics['appinfo_migration_end'] = get_current_time()
+ break
+ self.logger.info('AppInfo Migration Ended.')
+
+
+ # Reindex management app
- # Perform system re-index (it will grab date from input if provided)
job = self.start_reindex()
self.metrics['reindex_start'] = get_current_time()
self.logger.info('Started Re-index. Job=[%s]', job)
@@ -202,33 +254,44 @@ class Migrate:
self.logger.info("Finished Re-index. Job=[%s]", job)
self.metrics['reindex_end'] = get_current_time()
- # Only when we do a delta migration do we run the full data migration (includes appinfo and entity data)
- if self.delta_migration:
- self.logger.info('Delta option provided. Performing full data migration...')
- if self.is_data_migrated():
- self.reset_data_migration()
- time.sleep(STATUS_INTERVAL_SECONDS)
- self.is_data_migrated()
+ # Dedup and re-index all of organization's apps
- # self.start_core_data_migration()
- self.start_fulldata_migration()
+ app_ids = self.get_app_ids()
+ for app_id in app_ids:
- self.metrics['full_data_migration_start'] = get_current_time()
- self.logger.info("Full Data Migration Started")
- is_migrated = False
- while not is_migrated:
+ # De-dep app
+ job = self.start_app_dedup(app_id)
+ self.metrics['dedup_start_' + app_id] = get_current_time()
+ self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
+ is_running = True
+ while is_running:
time.sleep(STATUS_INTERVAL_SECONDS)
- is_migrated = self.is_data_migrated()
- if is_migrated:
+ is_running = self.is_reindex_running(job)
+ if not is_running:
break
- self.metrics['full_data_migration_end'] = get_current_time()
- self.logger.info("Full Data Migration completed")
+ self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
+ self.metrics['dedup_end_' + app_id] = get_current_time()
+
+ # Re-index app
+ job = self.start_app_reindex(app_id)
+ self.metrics['reindex_start_' + app_id] = get_current_time()
+ self.logger.info('Started Re-index. App=[%s], Job=[%s]', app_id, job)
+ is_running = True
+ while is_running:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_running = self.is_reindex_running(job)
+ if not is_running:
+ break
+
+ self.logger.info("Finished Re-index. App=[%s], Job=[%s]", app_id, job)
+ self.metrics['reindex_end_' + app_id] = get_current_time()
self.log_metrics()
self.logger.info("Finished...")
+
except KeyboardInterrupt:
self.log_metrics()
self.logger.error('Keyboard interrupted migration. Please run again to ensure the migration finished.')
@@ -237,6 +300,10 @@ class Migrate:
url = self.endpoint + '/system/database/setup'
return url
+ def get_database_bootstrap_url(self):
+ url = self.endpoint + '/system/database/bootstrap'
+ return url
+
def get_migration_url(self):
url = self.endpoint + '/system/migrate/run'
return url
@@ -249,6 +316,10 @@ class Migrate:
url = self.endpoint + '/system/migrate/status'
return url
+ def get_dedup_url(self):
+ url = self.endpoint + '/system/connection/dedup'
+ return url
+
def get_reindex_url(self):
url = self.endpoint + '/system/index/rebuild'
return url
@@ -257,7 +328,6 @@ class Migrate:
url = self.get_reindex_url() + "/management"
return url
-
def start_core_data_migration(self):
try:
r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
@@ -267,7 +337,6 @@ class Migrate:
self.logger.error('Failed to start migration, %s', e)
exit_on_error(str(e))
-
def start_fulldata_migration(self):
try:
r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
@@ -279,7 +348,7 @@ class Migrate:
def start_migration_system_update(self):
try:
- #TODO fix this URL
+ # TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
response = r.json()
@@ -299,6 +368,17 @@ class Migrate:
self.logger.error('Failed to run database setup, %s', e)
exit_on_error(str(e))
+ def run_database_bootstrap(self):
+ try:
+ setupUrl = self.get_database_bootstrap_url()
+ r = requests.put(url=setupUrl, auth=(self.admin_user, self.admin_pass))
+ if r.status_code != 200:
+ exit_on_error('Database Bootstrap Failed')
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to run database bootstrap, %s', e)
+ exit_on_error(str(e))
+
def start_index_mapping_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
@@ -437,7 +517,7 @@ class Migrate:
self.logger.error('Failed to get reindex status, %s', e)
# exit_on_error()
- def start_reindex(self):
+ def start_app_reindex(self, appId):
body = ""
if self.start_date is not None:
body = json.dumps({'updated': self.start_date})
@@ -463,6 +543,39 @@ class Migrate:
else:
return False
+ def get_dedup_status(self, job):
+ status_url = self.get_dedup_url()+'/' + job
+ try:
+ r = requests.get(url=status_url, auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response['status']
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to get dedup status, %s', e)
+ # exit_on_error()
+
+ def start_dedup(self, app_id):
+ body = ""
+ try:
+ r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.admin_user, self.admin_pass))
+ if r.status_code == 200:
+ response = r.json()
+ return response['jobId']
+ else:
+ self.logger.error('Failed to start dedup, %s', r)
+ exit_on_error(str(r))
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to make API request for dedup, %s', e)
+ exit_on_error(str(e))
+
+ def is_dedup_running(self, job):
+ status = self.get_dedup_status(job)
+ self.logger.info('Dedup status=[%s]', status)
+ if status != "COMPLETE":
+ return True
+ else:
+ return False
+
def is_endpoint_available(self):
try:
@@ -490,6 +603,24 @@ class Migrate:
)
+ def get_app_ids(self):
+
+ try:
+ url = self.endpoint + "/management/orgs/" + self.org + "/apps"
+ r = requests.get(url=url)
+ if r.status_code != 200:
+ exit_on_error('Database Bootstrap Failed')
+
+ apps = r.json()["data"]
+ app_ids = []
+ for appId in apps.values():
+ app_ids.append(appId)
+
+ return app_ids;
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to get list of application ids, %s', e)
+ exit_on_error(str(e))
def get_current_time():
return str(int(time.time()*1000))
[09/50] [abbrv] usergrid git commit: Added more logging statements
Posted by to...@apache.org.
Added more logging statements
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d35fea56
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d35fea56
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d35fea56
Branch: refs/heads/master
Commit: d35fea5636acfd384f0b35069cdc59e515ec0372
Parents: 8b4faf7
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 23 10:04:21 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 23 10:04:21 2015 -0600
----------------------------------------------------------------------
.../corepersistence/results/ObservableQueryExecutor.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d35fea56/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
index 7b31d19..7a32dce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
@@ -23,6 +23,9 @@ package org.apache.usergrid.corepersistence.results;
import java.util.Iterator;
import java.util.NoSuchElementException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.persistence.Results;
@@ -39,6 +42,7 @@ import rx.Observable;
public abstract class ObservableQueryExecutor<T> implements QueryExecutor {
+ private static final Logger logger = LoggerFactory.getLogger( ObservableQueryExecutor.class );
private Results results;
private Optional<String> cursor;
@@ -131,9 +135,12 @@ public abstract class ObservableQueryExecutor<T> implements QueryExecutor {
observable = buildNewResultsPage( cursor ).map( resultsPage -> createResultsInternal( resultsPage ) ).defaultIfEmpty(
new Results() );
+ logger.trace( "Trying to load results page" );
//take the first from our observable
final Results resultsPage = observable.take(1).toBlocking().first();
+ logger.trace( "Results page loaded {}", resultsPage );
+
//set the results for the iterator
this.results = resultsPage;
[18/50] [abbrv] usergrid git commit: This closes #409 -- invalid PR
Posted by to...@apache.org.
This closes #409 -- invalid PR
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/07d2ad37
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/07d2ad37
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/07d2ad37
Branch: refs/heads/master
Commit: 07d2ad3721d0ba64eddb4a899fb1348f3a5ff1b8
Parents: 5eed978
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Oct 26 14:47:35 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Oct 26 14:47:35 2015 -0600
----------------------------------------------------------------------
----------------------------------------------------------------------
[06/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/407/head' of apache.github.com:apache/usergrid into 2.1-release
Posted by to...@apache.org.
Merge commit 'refs/pull/407/head' of apache.github.com:apache/usergrid into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0604247f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0604247f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0604247f
Branch: refs/heads/master
Commit: 0604247f690db09b1a25feef9532cb5e58929437
Parents: 1fe1d1a 48161a1
Author: Mike Dunker <md...@apigee.com>
Authored: Thu Oct 22 11:27:59 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Thu Oct 22 11:27:59 2015 -0700
----------------------------------------------------------------------
.../corepersistence/index/IndexServiceImpl.java | 3 +-
.../persistence/core/astyanax/CassandraFig.java | 2 +-
.../core/astyanax/MultiRowColumnIterator.java | 46 +--
.../usergrid/persistence/graph/GraphFig.java | 2 +-
.../persistence/graph/guice/GraphModule.java | 11 +-
.../impl/EdgeSerializationImpl.java | 17 +-
.../impl/shard/AsyncTaskExecutor.java | 34 ++
.../graph/serialization/impl/shard/Shard.java | 15 +
.../impl/shard/ShardEntryGroup.java | 13 +-
.../impl/shard/ShardGroupCompaction.java | 4 -
.../impl/shard/ShardGroupDeletion.java | 78 ++++
.../impl/shard/impl/AsyncTaskExecutorImpl.java | 53 +++
.../shard/impl/NodeShardAllocationImpl.java | 81 ++--
.../shard/impl/ShardGroupColumnIterator.java | 72 ++--
.../shard/impl/ShardGroupCompactionImpl.java | 10 +-
.../impl/shard/impl/ShardGroupDeletionImpl.java | 219 +++++++++++
.../impl/shard/impl/ShardsColumnIterator.java | 10 +
.../graph/GraphManagerShardConsistencyIT.java | 378 ++++++++++++++-----
.../impl/shard/ShardEntryGroupTest.java | 14 +
.../impl/shard/ShardGroupCompactionTest.java | 30 +-
.../shard/impl/ShardGroupDeletionImplTest.java | 341 +++++++++++++++++
.../index/impl/EsEntityIndexImpl.java | 45 ++-
.../usergrid/persistence/queue/QueueFig.java | 2 +-
.../rest/applications/ApplicationResource.java | 50 +++
.../cassandra/ManagementServiceImpl.java | 3 +-
25 files changed, 1307 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0604247f/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
----------------------------------------------------------------------
[35/50] [abbrv] usergrid git commit: Adds the ability to get
credentials info as superuser for user migration
Posted by to...@apache.org.
Adds the ability to get credentials info as superuser for user migration
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/08e97813
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/08e97813
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/08e97813
Branch: refs/heads/master
Commit: 08e978132375751caf90ebcffde7a20d5e84b206
Parents: 5ed8c7c
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Oct 28 15:27:36 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 13:55:38 2015 -0600
----------------------------------------------------------------------
.../rest/applications/users/UserResource.java | 45 ++++++++++++++++++--
.../usergrid/management/ManagementService.java | 2 +
.../cassandra/ManagementServiceImpl.java | 19 +++++++++
3 files changed, 62 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/08e97813/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index df88cf0..fb10245 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -42,6 +42,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.exception.OAuthProblemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.management.ActivationState;
import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.EntityManager;
@@ -52,13 +57,10 @@ import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.tokens.exceptions.TokenException;
-import org.apache.amber.oauth2.common.exception.OAuthProblemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.commons.lang.StringUtils;
-
import com.sun.jersey.api.json.JSONWithPadding;
import com.sun.jersey.api.view.Viewable;
@@ -165,6 +167,41 @@ public class UserResource extends ServiceResource {
return new JSONWithPadding( response, callback );
}
+ @GET
+ @RequireSystemAccess
+ @Path("password")
+ public JSONWithPadding getUserPassword(@QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ logger.info( "UserResource.setUserPassword" );
+
+
+ final ApiResponse response = createApiResponse();
+ response.setAction( "get user password" );
+
+ final UUID applicationId = getApplicationId();
+ final UUID targetUserId = getUserUuid();
+
+ if ( applicationId == null ) {
+ response.setError( "Application not found" );
+ return new JSONWithPadding( response, callback );
+ }
+
+ if ( targetUserId == null ) {
+ response.setError( "User not found" );
+ return new JSONWithPadding( response, callback );
+ }
+
+ final CredentialsInfo credentialsInfo = management.getAppUserPasswordRaw( applicationId, targetUserId );
+
+
+ response.setProperty( "credentials", credentialsInfo );
+
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
@PUT
@Path("credentials")
http://git-wip-us.apache.org/repos/asf/usergrid/blob/08e97813/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index d69de2e..3f02e5a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -288,6 +288,8 @@ public interface ManagementService {
public void setAppUserPassword( UUID applicationId, UUID userId, String oldPassword, String newPassword )
throws Exception;
+ CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final UUID userId ) throws Exception;
+
/**
* Set the credentials info into the
* @param applicationId
http://git-wip-us.apache.org/repos/asf/usergrid/blob/08e97813/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 2e33539..b633727 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -2825,6 +2825,25 @@ public class ManagementServiceImpl implements ManagementService {
@Override
+ public CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final UUID userId ) throws Exception {
+
+ final User user = emf.getEntityManager( applicationId ).get( userId, User.class );
+
+ if(user == null){
+ throw new EntityNotFoundException("Could not find user with id " + userId + " in application" + applicationId );
+ }
+
+ final CredentialsInfo ci = readUserPasswordCredentials( applicationId, userId );
+
+ if ( ci == null ) {
+ throw new EntityNotFoundException("Could not find credentials for user with id " + userId + " in application" + applicationId );
+ }
+
+ return ci;
+ }
+
+
+ @Override
public User verifyAppUserPasswordCredentials( UUID applicationId, String name, String password ) throws Exception {
User user = findUserEntity( applicationId, name );
[17/50] [abbrv] usergrid git commit: Set the default shard size to
50k, not 500k.
Posted by to...@apache.org.
Set the default shard size to 50k, not 500k.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5eed9787
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5eed9787
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5eed9787
Branch: refs/heads/master
Commit: 5eed9787a820f2856ec9bdccac2caa710460d9ad
Parents: b025dda
Author: Michael Russo <mi...@gmail.com>
Authored: Mon Oct 26 10:54:34 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Mon Oct 26 10:54:34 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/persistence/graph/GraphFig.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5eed9787/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
index f0df7ff..ecb5b19 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
@@ -109,7 +109,7 @@ public interface GraphFig extends GuicyFig {
double getShardRepairChance();
- @Default( "500000" )
+ @Default( "50000" )
@Key( SHARD_SIZE )
long getShardSize();
[38/50] [abbrv] usergrid git commit: Adds test to prove functionality
of GET + PUT
Posted by to...@apache.org.
Adds test to prove functionality of GET + PUT
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ba629da5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ba629da5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ba629da5
Branch: refs/heads/master
Commit: ba629da577235c0c9bedf4addcb44bfb5fdbdccb
Parents: 4784b34
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 29 15:25:42 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 15:25:42 2015 -0600
----------------------------------------------------------------------
.../collection/users/UserResourceIT.java | 109 +++++++++++++++++++
1 file changed, 109 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ba629da5/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index 79b8c85..deda5eb 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@ -37,9 +37,15 @@ import org.apache.usergrid.rest.applications.utils.UserRepo;
import org.apache.usergrid.utils.UUIDUtils;
import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
+
import java.io.IOException;
+import javax.ws.rs.core.MediaType;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -1103,4 +1109,107 @@ public class UserResourceIT extends AbstractRestIT {
assertEquals(response.getResponse().getEntities().get(0).get("uuid").toString(), userId.toString());
}
+
+
+
+ @Test
+ public void testCredentialsTransfer() throws Exception {
+
+ usersResource.post(new User("test_1", "Test1 User", "test_1@test.com", "test123")); // client.setApiUrl(apiUrl);
+ refreshIndex();
+
+ //Entity appInfo = this.app().get().getResponse().getEntities().get(0);
+
+ Token token = this.app().token().post(new Token("test_1", "test123"));
+
+ assertNotNull(token.getAccessToken());
+
+ final String superUserName = this.clientSetup.getSuperuserName();
+ final String superUserPassword = this.clientSetup.getSuperuserPassword();
+
+
+ //get the credentials info
+ final CollectionEndpoint collection = userResource.entity("test_1").collection( "credentials" );
+
+ final WebResource resource = collection.getResource();
+
+ resource.addFilter( new HTTPBasicAuthFilter(superUserName, superUserPassword) );
+
+
+
+ final ApiResponse response = resource.type( MediaType.APPLICATION_JSON_TYPE)
+ .accept( MediaType.APPLICATION_JSON ).get( org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
+
+ //now get the credentials sub object
+
+ final Map<String, Object> credentials = ( Map<String, Object> ) response.getProperties().get( "credentials" );
+
+
+
+ //get out the hash and change it so we can validate
+ final String originalSecret = ( String ) credentials.get( "secret" );
+
+
+ //here we modify the hash a little, this way we can break password validation, then re-set it to ensure we're actually updating the credentials info correctly.
+ final String borkedSecret = originalSecret.substring( 0, originalSecret.length() -1 );
+
+ credentials.put( "credentials", borkedSecret );
+ credentials.put( "secret", borkedSecret );
+
+ //now PUT it
+
+
+ final Map<String, Map<String, Object>> wrapper = new HashMap<>( );
+ wrapper.put( "credentials", credentials );
+
+ final WebResource putResource = collection.getResource();
+
+ putResource.addFilter( new HTTPBasicAuthFilter(superUserName, superUserPassword) );
+
+
+ putResource.type( MediaType.APPLICATION_JSON_TYPE)
+ .accept( MediaType.APPLICATION_JSON ).put(
+ org.apache.usergrid.rest.test.resource.model.ApiResponse.class, wrapper );
+
+
+ //now try to get a password, it should fail because the hash is no longer correct
+
+ int status = 0;
+
+ // bad access token
+ try {
+ this.app().token().post(new Token("test_1", "test123"));
+ fail("Should have thrown an exception");
+ } catch (UniformInterfaceException uie) {
+ status = uie.getResponse().getStatus();
+ log.info("Error Response Body: " + uie.getResponse().getEntity(String.class));
+ }
+
+ assertEquals(Status.BAD_REQUEST.getStatusCode(), status);
+
+
+ //now put the correct one
+
+
+ credentials.put( "credentials", originalSecret );
+ credentials.put( "secret", originalSecret );
+
+
+ final WebResource putResource2 = collection.getResource();
+
+ putResource2.addFilter( new HTTPBasicAuthFilter( superUserName, superUserPassword ) );
+
+
+ putResource2.type( MediaType.APPLICATION_JSON_TYPE)
+ .accept( MediaType.APPLICATION_JSON ).put(
+ org.apache.usergrid.rest.test.resource.model.ApiResponse.class, wrapper );
+
+
+ //now auth, should be good
+ final Token nextToken = this.app().token().post(new Token("test_1", "test123"));
+
+ assertNotNull( nextToken.getAccessToken() );
+
+ }
}
[24/50] [abbrv] usergrid git commit: Moves system endpoints into the
system package
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
new file mode 100644
index 0000000..c5a6dbc
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.rest.system;
+
+
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Preconditions;
+import com.google.inject.Injector;
+import com.sun.jersey.api.json.JSONWithPadding;
+
+
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON,
+ "application/javascript",
+ "application/x-javascript",
+ "text/ecmascript",
+ "application/ecmascript",
+ "text/jscript"
+} )
+public class MigrateResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( MigrateResource.class );
+
+ public MigrateResource() {
+ logger.info( "SystemResource initialized" );
+ }
+
+ @Autowired
+ private Injector guiceInjector;
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "run" )
+ public JSONWithPadding migrateData( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data" );
+ //TODO make this use the task scheduler
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+
+ logger.info( "Migrating Schema" );
+
+ try {
+ getMigrationManager().migrate();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data", e );
+ }
+
+ logger.info( "Migrating Data" );
+
+ try {
+ getDataMigrationManager().migrate();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data", e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats" );
+ migrate.setDaemon( true );
+ migrate.start();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "run/{pluginName}" )
+ public JSONWithPadding migrateData(@PathParam("pluginName") String pluginName , @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ if(!getDataMigrationManager().pluginExists(pluginName)){
+ throw new IllegalArgumentException("Plugin doesn't exits name:"+pluginName);
+ }
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data: "+ pluginName );
+ //TODO make this use the task scheduler
+
+
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+
+ logger.info( "Migrating Data for plugin: " + pluginName );
+
+ try {
+ getDataMigrationManager().migrate(pluginName);
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data for plugin: " + pluginName, e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats: "+pluginName );
+ migrate.setDaemon( true );
+ migrate.start();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "set" )
+ public JSONWithPadding setMigrationVersion(
+ @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback )
+ throws Exception {
+
+ logger.debug( "setMigrationVersion" );
+
+ Preconditions.checkNotNull( json, "You must provide a json body" );
+ Preconditions.checkArgument( json.keySet().size() > 0, "You must specify at least one module and version" );
+
+ ApiResponse response = createApiResponse();
+ response.setAction("Set Migration Versions");
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ /**
+ * Set the migration version for the plugins specified
+ */
+ for ( final String key : json.keySet() ) {
+
+ int version = ( int ) json.get( key );
+
+ dataMigrationManager.resetToVersion(key, version);
+ }
+
+
+ /**
+ * Echo back a response of the current versions for all plugins
+ */
+ for(final String pluginName: plugins){
+ node.put(pluginName, dataMigrationManager.getCurrentVersion(pluginName));
+ }
+
+
+ response.setData( node );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "status" )
+ public JSONWithPadding migrateStatus(
+ @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Schema indexes" );
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+
+
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ for(final String pluginName: plugins){
+ node.put( pluginName, dataMigrationManager.getCurrentVersion( pluginName ) );
+ }
+
+ response.setData( node );
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "count" )
+ public JSONWithPadding migrateCount(
+ @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Current entity count in system" );
+
+ response.setProperty( "count", emf.performEntityCount() );
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ /**
+ * Get the schema migration manager
+ */
+ private MigrationManager getMigrationManager() {
+ return guiceInjector.getInstance( MigrationManager.class );
+ }
+
+ /**
+ * Get the Data migration manager
+ */
+ private DataMigrationManager getDataMigrationManager() {
+ return guiceInjector.getInstance( DataMigrationManager.class );
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
new file mode 100644
index 0000000..a5174e6
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.system;
+
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.sun.jersey.api.json.JSONWithPadding;
+
+
+@Path( "/system" )
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class SystemResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( SystemResource.class );
+
+
+ public SystemResource() {
+ logger.info( "SystemResource initialized" );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "superuser/setup" )
+ public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "superuser setup" );
+
+ logger.info( "Setting up Superuser" );
+
+ try {
+ management.provisionSuperuser();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to complete superuser setup", e );
+ }
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+
+ @Path( "migrate" )
+ public MigrateResource migrate() {
+ return getSubResource( MigrateResource.class );
+ }
+
+
+ @Path( "index" )
+ public IndexResource index() { return getSubResource( IndexResource.class ); }
+
+
+ @Path( "database" )
+ public DatabaseResource database() {
+ return getSubResource( DatabaseResource.class );
+ }
+
+ @Path( "applications" )
+ public ApplicationsResource applications() {
+ return getSubResource( ApplicationsResource.class );
+ }
+
+
+ @Path( "connection" )
+ public ConnectionResource connection() { return getSubResource( ConnectionResource.class ); }
+
+}
[08/50] [abbrv] usergrid git commit: Removed unused class and added
logging
Posted by to...@apache.org.
Removed unused class and added logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8b4faf72
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8b4faf72
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8b4faf72
Branch: refs/heads/master
Commit: 8b4faf72221ec9eb2b8bb59cca0263c20b483ccb
Parents: db852d0
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 23 09:58:45 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 23 09:58:45 2015 -0600
----------------------------------------------------------------------
.../read/traverse/AbstractReadGraphFilter.java | 10 ++-
.../read/traverse/EntityLoadVerifyFilter.java | 24 +++---
.../persistence/ObservableIterator.java | 83 --------------------
.../persistence/core/rx/ObservableIterator.java | 2 +-
4 files changed, 24 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8b4faf72/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index f477092..d3e0345 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,6 +20,9 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
@@ -41,6 +44,8 @@ import rx.Observable;
*/
public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id, Edge> {
+ private static final Logger logger = LoggerFactory.getLogger( AbstractReadGraphFilter.class );
+
private final GraphManagerFactory graphManagerFactory;
@@ -82,7 +87,10 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
*/
return graphManager.loadEdgesFromSource( search )
//set the edge state for cursors
- .doOnNext( edge -> edgeCursorState.update( edge ) )
+ .doOnNext( edge -> {
+ logger.trace( "Seeking over edge {}", edge );
+ edgeCursorState.update( edge );
+ } )
//map our id from the target edge and set our cursor every edge we traverse
.map( edge -> createFilterResult( edge.getTargetNode(), edgeCursorState.getCursorEdge(),
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8b4faf72/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java
index 41507e9..c782bce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java
@@ -47,7 +47,9 @@ import rx.Observable;
*
* TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
*/
-public class EntityLoadVerifyFilter extends AbstractFilter<FilterResult<Id>, FilterResult<Entity>>{
+public class EntityLoadVerifyFilter extends AbstractFilter<FilterResult<Id>, FilterResult<Entity>> {
+
+ private static final Logger logger = LoggerFactory.getLogger( EntityLoadVerifyFilter.class );
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
@@ -69,17 +71,19 @@ public class EntityLoadVerifyFilter extends AbstractFilter<FilterResult<Id>, Fil
final Observable<FilterResult<Entity>> entityObservable =
filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( bufferedIds -> {
- final Observable<EntitySet> entitySetObservable =
- Observable.from( bufferedIds ).map( filterResultId -> filterResultId.getValue() ).toList()
- .flatMap( ids -> entityCollectionManager.load( ids ) );
+ logger.trace( "Attempting to batch load ids {}", bufferedIds );
+
+ final Observable<EntitySet> entitySetObservable =
+ Observable.from( bufferedIds ).map( filterResultId -> filterResultId.getValue() ).toList()
+ .flatMap( ids -> entityCollectionManager.load( ids ) );
- //now we have a collection, validate our canidate set is correct.
+ //now we have a collection, validate our canidate set is correct.
- return entitySetObservable.map( entitySet -> new EntityVerifier( entitySet, bufferedIds ) )
- .doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
- entityCollector -> Observable.from( entityCollector.getResults() ) );
- } );
+ return entitySetObservable.map( entitySet -> new EntityVerifier( entitySet, bufferedIds ) )
+ .doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
+ entityCollector -> Observable.from( entityCollector.getResults() ) );
+ } );
return entityObservable;
}
@@ -132,7 +136,7 @@ public class EntityLoadVerifyFilter extends AbstractFilter<FilterResult<Id>, Fil
//doesn't exist warn and drop
if ( entity == null || !entity.getEntity().isPresent() ) {
logger.warn( "Read graph edge and received candidate with entityId {}, yet was not found in cassandra."
- + " Ignoring since this could be a region sync issue", candidateId );
+ + " Ignoring since this could be a region sync issue", candidateId );
//TODO trigger an audit after a fail count where we explicitly try to repair from other regions
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8b4faf72/stack/core/src/main/java/org/apache/usergrid/persistence/ObservableIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/ObservableIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/ObservableIterator.java
deleted file mode 100644
index 9befb79..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/ObservableIterator.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.persistence;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import rx.Observable;
-import rx.Subscriber;
-
-import java.util.Iterator;
-
-
-/**
- * Converts an iterator to an observable. Subclasses need to only implement getting the iterator from the data source.
- * This is used in favor of "Observable.just" when the initial fetch of the iterator will require I/O. This allows us
- * to wrap the iterator in a deferred invocation to avoid the blocking on construction.
- */
-public abstract class ObservableIterator<T> implements Observable.OnSubscribe<T> {
-
- private static final Logger log = LoggerFactory.getLogger(ObservableIterator.class);
-
- private final String name;
-
-
- /**
- * @param name The simple name of the iterator, used for debugging
- */
- protected ObservableIterator( final String name ) {this.name = name;}
-
-
- @Override
- public void call( final Subscriber<? super T> subscriber ) {
-
-
- try {
- //get our iterator and push data to the observer
- final Iterator<T> itr = getIterator();
-
- Preconditions.checkNotNull(itr,
- "The observable must return an iterator. Null was returned for iterator " + name);
-
-
- //while we have items to emit and our subscriber is subscribed, we want to keep emitting items
- while ( itr.hasNext() && !subscriber.isUnsubscribed() ) {
- final T next = itr.next();
-
- log.trace( "Iterator '{}' emitting item '{}'", name, next );
-
- subscriber.onNext( next );
- }
-
-
- subscriber.onCompleted();
- }
-
- //if any error occurs, we need to notify the observer so it can perform it's own error handling
- catch ( Throwable t ) {
- log.error( "Unable to emit items from iterator {}", name, t );
- subscriber.onError( t );
- }
- }
-
-
- /**
- * Return the iterator to feed data to
- */
- protected abstract Iterator<T> getIterator();
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8b4faf72/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java
index 84a7fc3..57409e1 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java
@@ -66,7 +66,7 @@ public abstract class ObservableIterator<T> implements Observable.OnSubscribe<T>
while ( itr.hasNext() && !subscriber.isUnsubscribed() ) {
final T next = itr.next();
-// log.trace( "Iterator '{}' emitting item '{}'", name, next );
+ log.trace( "Iterator '{}' emitting item '{}'", name, next );
subscriber.onNext( next );
}
[50/50] [abbrv] usergrid git commit: Fixes race condition in test
Posted by to...@apache.org.
Fixes race condition in test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b031d4d3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b031d4d3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b031d4d3
Branch: refs/heads/master
Commit: b031d4d3da09ee65dcde5ab7521627bb2782c9ea
Parents: b422364
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 30 17:32:19 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 30 17:32:19 2015 -0600
----------------------------------------------------------------------
.../java/org/apache/usergrid/corepersistence/index/RxTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b031d4d3/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
index d7b0bdb..ce78b15 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/RxTest.java
@@ -45,10 +45,10 @@ public class RxTest {
final int count = 10;
- final CountDownLatch latch = new CountDownLatch( count );
+ final CountDownLatch latch = new CountDownLatch( count+1 );
final Subscription connectedObservable =
- Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).subscribeOn( Schedulers.io() )
+ Observable.range( 0, count ).doOnNext( integer -> latch.countDown() ).doOnCompleted( () -> latch.countDown() ).subscribeOn( Schedulers.io() )
.subscribe();
[13/50] [abbrv] usergrid git commit: Changed the interface to return
MarkedEdge interface. This allows the caller to determine if they want to
filter or not, as well as to perform read-repair actions on read.
Posted by to...@apache.org.
Changed the interface to return MarkedEdge interface. This allows the caller to determine if they want to filter or not, as well as to perform read-repair actions on read.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1d4f22c5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1d4f22c5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1d4f22c5
Branch: refs/heads/master
Commit: 1d4f22c59c057a6f9d52844e03897786b0773a5b
Parents: 4e51d38
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 23 15:25:39 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 23 15:25:39 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 7 +-
.../corepersistence/CpRelationManager.java | 16 ++---
.../read/traverse/AbstractReadGraphFilter.java | 74 +++++++++++++++----
.../read/traverse/EdgeCursorSerializer.java | 8 ++-
.../traverse/ReadGraphCollectionFilter.java | 5 +-
.../ReadGraphConnectionByTypeFilter.java | 11 +--
.../traverse/ReadGraphConnectionFilter.java | 5 +-
.../index/AsyncIndexServiceTest.java | 3 +-
.../corepersistence/index/IndexServiceTest.java | 12 ++--
.../pipeline/cursor/CursorTest.java | 24 +++----
.../service/ConnectionServiceImplTest.java | 5 +-
.../persistence/ApplicationServiceIT.java | 4 +-
.../persistence/graph/GraphManager.java | 14 ++--
.../usergrid/persistence/graph/MarkedEdge.java | 1 +
.../graph/impl/GraphManagerImpl.java | 30 ++++----
.../graph/impl/SimpleMarkedEdge.java | 39 ++++++----
.../persistence/graph/GraphManagerIT.java | 76 ++++++++++----------
.../persistence/graph/GraphManagerLoadTest.java | 10 +--
.../graph/GraphManagerShardConsistencyIT.java | 6 +-
.../graph/GraphManagerStressTest.java | 16 ++---
.../management/AppInfoMigrationPlugin.java | 5 +-
21 files changed, 220 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 4bdade5..c75a025 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -63,6 +63,7 @@ import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.EntityIndex;
@@ -477,9 +478,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}",
new Object[]{edgeType, managementId.getType(), managementId.getUuid()});
- Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
- managementId, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ));
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSource(
+ new SimpleSearchByEdgeType( managementId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ Optional.<Edge>absent() ) );
final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( appScope );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index aad7610..b4cabc4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -59,6 +59,7 @@ import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
@@ -227,14 +228,9 @@ public class CpRelationManager implements RelationManager {
Observable<Edge> edges =
gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeType, null ) )
- .flatMap( new Func1<String, Observable<Edge>>() {
- @Override
- public Observable<Edge> call( final String edgeType ) {
- return gm.loadEdgesToTarget(
- new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
- }
- } );
+ .flatMap( edgeType1 -> gm.loadEdgesToTarget(
+ new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType1, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) ) );
//if our limit is set, take them. Note this logic is still borked, we can't possibly fit everything in memmory
if ( limit > -1 ) {
@@ -268,7 +264,7 @@ public class CpRelationManager implements RelationManager {
} );
GraphManager gm = managerCache.getGraphManager( applicationScope );
- Observable<Edge> edges = gm.loadEdgeVersions( CpNamingUtils
+ Observable<MarkedEdge> edges = gm.loadEdgeVersions( CpNamingUtils
.createEdgeFromConnectionType( new SimpleId( headEntity.getUuid(), headEntity.getType() ), connectionType,
entityId ) );
@@ -288,7 +284,7 @@ public class CpRelationManager implements RelationManager {
} );
GraphManager gm = managerCache.getGraphManager( applicationScope );
- Observable<Edge> edges = gm.loadEdgeVersions( CpNamingUtils
+ Observable<MarkedEdge> edges = gm.loadEdgeVersions( CpNamingUtils
.createEdgeFromCollectionName( new SimpleId( headEntity.getUuid(), headEntity.getType() ), collectionName,
entityId ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index d3e0345..621edd2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -23,13 +23,16 @@ package org.apache.usergrid.corepersistence.pipeline.read.traverse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -42,18 +45,21 @@ import rx.Observable;
/**
* Command for reading graph edges
*/
-public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id, Edge> {
+public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id, MarkedEdge> {
private static final Logger logger = LoggerFactory.getLogger( AbstractReadGraphFilter.class );
private final GraphManagerFactory graphManagerFactory;
+ private final AsyncEventService asyncEventService;
/**
* Create a new instance of our command
*/
- public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory ) {
+ public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory,
+ final AsyncEventService asyncEventService ) {
this.graphManagerFactory = graphManagerFactory;
+ this.asyncEventService = asyncEventService;
}
@@ -61,9 +67,11 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> previousIds ) {
+ final ApplicationScope applicationScope = pipelineContext.getApplicationScope();
+
//get the graph manager
final GraphManager graphManager =
- graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() );
+ graphManagerFactory.createEdgeManager( applicationScope );
final String edgeName = getEdgeTypeName();
@@ -74,18 +82,60 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
return previousIds.flatMap( previousFilterValue -> {
//set our our constant state
- final Optional<Edge> startFromCursor = getSeekValue();
+ final Optional<MarkedEdge> startFromCursor = getSeekValue();
final Id id = previousFilterValue.getValue();
+ final Optional<Edge> typeWrapper = Optional.fromNullable(startFromCursor.orNull());
+
+ /**
+ * We do not want to filter. This is intentional DO NOT REMOVE!!!
+ *
+ * We want to fire events on these edges if they exist, the delete was missed.
+ */
final SimpleSearchByEdgeType search =
new SimpleSearchByEdgeType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- startFromCursor );
+ typeWrapper, false );
/**
* TODO, pass a message with pointers to our cursor values to be generated later
*/
- return graphManager.loadEdgesFromSource( search )
+ return graphManager.loadEdgesFromSource( search ).filter(markedEdge -> {
+
+ final boolean isDeleted = markedEdge.isDeleted();
+ final boolean isSourceNodeDeleted = markedEdge.isSourceNodeDelete();
+ final boolean isTargetNodeDelete = markedEdge.isTargetNodeDeleted();
+
+
+
+ if(isDeleted){
+ logger.trace( "Edge {} is deleted, queueing the delete event", markedEdge );
+ asyncEventService.queueDeleteEdge( applicationScope, markedEdge );
+ }
+
+ if(isSourceNodeDeleted){
+ final Id sourceNodeId = markedEdge.getSourceNode();
+
+ logger.trace( "Edge {} has a deleted source node, queueing the delete entity event for id {}", markedEdge, sourceNodeId );
+
+ asyncEventService.queueEntityDelete( applicationScope, sourceNodeId );
+ }
+
+ if(isTargetNodeDelete){
+
+ final Id targetNodeId = markedEdge.getTargetNode();
+
+ logger.trace( "Edge {} has a deleted target node, queueing the delete entity event for id {}", markedEdge, targetNodeId );
+
+ asyncEventService.queueEntityDelete( applicationScope, targetNodeId );
+ }
+
+
+ //filter if any of them are marked
+ return !isDeleted && !isSourceNodeDeleted && !isTargetNodeDelete;
+
+
+ })
//set the edge state for cursors
.doOnNext( edge -> {
logger.trace( "Seeking over edge {}", edge );
@@ -100,7 +150,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
@Override
- protected FilterResult<Id> createFilterResult( final Id emit, final Edge cursorValue,
+ protected FilterResult<Id> createFilterResult( final Id emit, final MarkedEdge cursorValue,
final Optional<EdgePath> parent ) {
//if it's our first pass, there's no cursor to generate
@@ -113,7 +163,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
@Override
- protected CursorSerializer<Edge> getCursorSerializer() {
+ protected CursorSerializer<MarkedEdge> getCursorSerializer() {
return EdgeCursorSerializer.INSTANCE;
}
@@ -131,14 +181,14 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
*/
private final class EdgeState {
- private Edge cursorEdge = null;
- private Edge currentEdge = null;
+ private MarkedEdge cursorEdge = null;
+ private MarkedEdge currentEdge = null;
/**
* Update the pointers
*/
- private void update( final Edge newEdge ) {
+ private void update( final MarkedEdge newEdge ) {
cursorEdge = currentEdge;
currentEdge = newEdge;
}
@@ -147,7 +197,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
/**
* Get the edge to use in cursors for resume
*/
- private Edge getCursorEdge() {
+ private MarkedEdge getCursorEdge() {
return cursorEdge;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java
index 8d9bf6f..d54e547 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java
@@ -22,20 +22,22 @@ package org.apache.usergrid.corepersistence.pipeline.read.traverse;
import org.apache.usergrid.corepersistence.pipeline.cursor.AbstractCursorSerializer;
import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
/**
* Edge cursor serializer
*/
-public class EdgeCursorSerializer extends AbstractCursorSerializer<Edge> {
+public class EdgeCursorSerializer extends AbstractCursorSerializer<MarkedEdge> {
public static final EdgeCursorSerializer INSTANCE = new EdgeCursorSerializer();
@Override
- protected Class<SimpleEdge> getType() {
- return SimpleEdge.class;
+ protected Class<SimpleMarkedEdge> getType() {
+ return SimpleMarkedEdge.class;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
index dc39f5c..db5a0a8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import com.google.inject.Inject;
@@ -40,8 +41,8 @@ public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String collectionName ) {
- super( graphManagerFactory );
+ public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, final AsyncEventService asyncEventService, @Assisted final String collectionName ) {
+ super( graphManagerFactory, asyncEventService );
this.collectionName = collectionName;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java
index 61ba4ad..054a52b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java
@@ -26,6 +26,7 @@ import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -42,7 +43,7 @@ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeType
/**
* Command for reading graph edges on a connection
*/
-public class ReadGraphConnectionByTypeFilter extends AbstractPathFilter<Id, Id, Edge>{
+public class ReadGraphConnectionByTypeFilter extends AbstractPathFilter<Id, Id, MarkedEdge>{
private final GraphManagerFactory graphManagerFactory;
private final String connectionName;
@@ -77,12 +78,14 @@ public class ReadGraphConnectionByTypeFilter extends AbstractPathFilter<Id, Id,
return filterResultObservable.flatMap( idFilterResult -> {
//set our our constant state
- final Optional<Edge> startFromCursor = getSeekValue();
+ final Optional<MarkedEdge> startFromCursor = getSeekValue();
final Id id = idFilterResult.getValue();
+ final Optional<Edge> typeWrapper = Optional.fromNullable(startFromCursor.orNull());
+
final SimpleSearchByIdType search =
new SimpleSearchByIdType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- entityType, startFromCursor );
+ entityType, typeWrapper );
return graphManager.loadEdgesFromSourceByType( search ).map(
edge -> createFilterResult( edge.getTargetNode(), edge, idFilterResult.getPath() ));
@@ -91,7 +94,7 @@ public class ReadGraphConnectionByTypeFilter extends AbstractPathFilter<Id, Id,
@Override
- protected CursorSerializer<Edge> getCursorSerializer() {
+ protected CursorSerializer<MarkedEdge> getCursorSerializer() {
return EdgeCursorSerializer.INSTANCE;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
index 11ec5f8..93e8fd4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import com.google.inject.Inject;
@@ -40,8 +41,8 @@ public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String connectionName ) {
- super( graphManagerFactory );
+ public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, final AsyncEventService asyncEventService, @Assisted final String connectionName ) {
+ super( graphManagerFactory, asyncEventService );
this.connectionName = connectionName;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
index 2863cbf..74f9ce0 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.UUID;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.index.*;
import org.junit.Before;
import org.junit.Rule;
@@ -135,7 +136,7 @@ public abstract class AsyncIndexServiceTest {
*/
- final List<Edge> connectionSearchEdges = Observable.range( 0, 500 ).flatMap(integer -> {
+ final List<MarkedEdge> connectionSearchEdges = Observable.range( 0, 500 ).flatMap(integer -> {
final Id connectingId = createId("connecting");
final Edge edge = CpNamingUtils.createConnectionEdge(connectingId, "likes", testEntity.getId());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
index 6001dd4..90d6c5a 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.junit.Before;
import org.junit.Test;
@@ -244,7 +245,7 @@ public class IndexServiceTest {
// final int edgeCount = indexFig.getIndexBatchSize()*2;
final int edgeCount = 100;
- final List<Edge> connectionSearchEdges = Observable.range( 0, edgeCount ).flatMap( integer -> {
+ final List<MarkedEdge> connectionSearchEdges = Observable.range( 0, edgeCount ).flatMap( integer -> {
final Id connectingId = createId( "connecting" );
final Edge edge = CpNamingUtils.createConnectionEdge( connectingId, "likes", testEntity.getId() );
@@ -377,7 +378,8 @@ public class IndexServiceTest {
//Write multiple connection edges
final int edgeCount = 5;
- final List<Edge> connectionSearchEdges = createConnectionSearchEdges( testEntity, graphManager, edgeCount );
+ final List<MarkedEdge>
+ connectionSearchEdges = createConnectionSearchEdges( testEntity, graphManager, edgeCount );
indexService.indexEntity( applicationScope, testEntity ).flatMap(mesage -> indexProducer.put(mesage)).toBlocking().getIterator();
@@ -485,10 +487,10 @@ public class IndexServiceTest {
}
- private List<Edge> createConnectionSearchEdges(
- final Entity testEntity, final GraphManager graphManager, final int edgeCount ) {
+ private List<MarkedEdge> createConnectionSearchEdges( final Entity testEntity, final GraphManager graphManager,
+ final int edgeCount ) {
- final List<Edge> connectionSearchEdges = Observable.range( 0, edgeCount ).flatMap( integer -> {
+ final List<MarkedEdge> connectionSearchEdges = Observable.range( 0, edgeCount ).flatMap( integer -> {
//create our connection edge.
final Id connectingId = createId( "connecting" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java
index 7128dcf..c9dcbf1 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java
@@ -28,7 +28,9 @@ import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
import org.apache.usergrid.corepersistence.pipeline.read.search.ElasticsearchCursorSerializer;
import org.apache.usergrid.corepersistence.pipeline.read.traverse.EdgeCursorSerializer;
import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
import com.google.common.base.Optional;
@@ -40,19 +42,12 @@ public class CursorTest {
@Test
public void testCursors(){
+ //test encoding edge
+ final MarkedEdge edge1 = new SimpleMarkedEdge( createId("source1"), "edgeType1", createId("target1"), 100, false, false, false );
-
-
-
-
- //test encoding edge
-
- final Edge edge1 = new SimpleEdge( createId("source1"), "edgeType1", createId("target1"), 100 );
-
-
- final Edge edge2 = new SimpleEdge( createId("source2"), "edgeType2", createId("target2"), 110 );
+ final MarkedEdge edge2 = new SimpleMarkedEdge( createId("source2"), "edgeType2", createId("target2"), 110, false, false, false );
@@ -64,11 +59,12 @@ public class CursorTest {
final EdgePath<Integer> filter3Path = new EdgePath<>( 3, query2, ElasticsearchCursorSerializer.INSTANCE, Optional.absent() );
- final EdgePath<Edge> filter2Path = new EdgePath<Edge>(2, edge2, EdgeCursorSerializer.INSTANCE, Optional.of( filter3Path ));
+ final EdgePath<MarkedEdge> filter2Path =
+ new EdgePath<>( 2, edge2, EdgeCursorSerializer.INSTANCE, Optional.of( filter3Path ) );
final EdgePath<Integer> filter1Path = new EdgePath<>( 1, query1, ElasticsearchCursorSerializer.INSTANCE, Optional.of(filter2Path) );
- final EdgePath<Edge> filter0Path = new EdgePath<>( 0, edge1, EdgeCursorSerializer.INSTANCE, Optional.of( filter1Path ) );
+ final EdgePath<MarkedEdge> filter0Path = new EdgePath<>( 0, edge1, EdgeCursorSerializer.INSTANCE, Optional.of( filter1Path ) );
@@ -91,7 +87,7 @@ public class CursorTest {
assertEquals(query2, parsedQuery2);
- final Edge parsedEdge2 = requestCursor.getCursor( 2, EdgeCursorSerializer.INSTANCE );
+ final MarkedEdge parsedEdge2 = requestCursor.getCursor( 2, EdgeCursorSerializer.INSTANCE );
assertEquals( edge2, parsedEdge2 );
@@ -100,7 +96,7 @@ public class CursorTest {
assertEquals( query1, parsedQuery1 );
- final Edge parsedEdge1 = requestCursor.getCursor( 0, EdgeCursorSerializer.INSTANCE );
+ final MarkedEdge parsedEdge1 = requestCursor.getCursor( 0, EdgeCursorSerializer.INSTANCE );
assertEquals(edge1, parsedEdge1);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
index 326e128..6929d87 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/service/ConnectionServiceImplTest.java
@@ -36,6 +36,7 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
@@ -118,7 +119,7 @@ public class ConnectionServiceImplTest {
new SimpleSearchByEdge( source, connectionEdge.getType(), target, Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING, Optional.absent() );
- final List<Edge> edges = gm.loadEdgeVersions( simpleSearchByEdge ).toList().toBlocking().last();
+ final List<MarkedEdge> edges = gm.loadEdgeVersions( simpleSearchByEdge ).toList().toBlocking().last();
assertEquals( 1, edges.size() );
@@ -209,7 +210,7 @@ public class ConnectionServiceImplTest {
SearchByEdgeType.Order.DESCENDING, Optional.absent() );
//check only 1 exists
- final List<Edge> edges = gm.loadEdgeVersions( simpleSearchByEdge ).toList().toBlocking().last();
+ final List<MarkedEdge> edges = gm.loadEdgeVersions( simpleSearchByEdge ).toList().toBlocking().last();
assertEquals( 1, edges.size() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
index d870114..658d3eb 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
@@ -30,6 +30,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
@@ -86,7 +87,8 @@ public class ApplicationServiceIT extends AbstractCoreIT {
, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent() );
- Iterator<Edge> results = graphManager.loadEdgesFromSource(simpleSearchByEdgeType).toBlocking().getIterator();
+ Iterator<MarkedEdge>
+ results = graphManager.loadEdgesFromSource(simpleSearchByEdgeType).toBlocking().getIterator();
if(results.hasNext()){
Assert.fail("should be empty");
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
index 6100725..000c633 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
@@ -59,7 +59,7 @@ public interface GraphManager extends CPManager {
* Create or update an edge. Note that the implementation should also create incoming (reversed) edges for this
* edge.
*/
- Observable<Edge> writeEdge( Edge edge );
+ Observable<MarkedEdge> writeEdge( Edge edge );
/**
@@ -68,7 +68,7 @@ public interface GraphManager extends CPManager {
*
* Implementation should also mark the incoming (reversed) edge. Only marks the specific version
*/
- Observable<Edge> markEdge( Edge edge );
+ Observable<MarkedEdge> markEdge( Edge edge );
/**
* @param edge Remove the edge in the graph
@@ -98,7 +98,7 @@ public interface GraphManager extends CPManager {
/**
* Get all versions of this edge where versions <= max version
*/
- Observable<Edge> loadEdgeVersions( SearchByEdge edge );
+ Observable<MarkedEdge> loadEdgeVersions( SearchByEdge edge );
/**
* Returns an observable that emits all edges where the specified node is the source node. The edges will match the
@@ -108,7 +108,7 @@ public interface GraphManager extends CPManager {
*
* @return An observable that emits Edges. The observer will need to unsubscribe when it has completed consumption.
*/
- Observable<Edge> loadEdgesFromSource( SearchByEdgeType search );
+ Observable<MarkedEdge> loadEdgesFromSource( SearchByEdgeType search );
/**
* Returns an observable that emits all edges where the specified node is the target node. The edges will match the
@@ -118,7 +118,7 @@ public interface GraphManager extends CPManager {
*
* @return An observable that emits Edges. The observer will need to unsubscribe when it has completed consumption.
*/
- Observable<Edge> loadEdgesToTarget( SearchByEdgeType search );
+ Observable<MarkedEdge> loadEdgesToTarget( SearchByEdgeType search );
/**
@@ -129,7 +129,7 @@ public interface GraphManager extends CPManager {
*
* @return An observable that emits Edges. The observer will need to unsubscribe when it has completed consumption.
*/
- Observable<Edge> loadEdgesFromSourceByType( SearchByIdType search );
+ Observable<MarkedEdge> loadEdgesFromSourceByType( SearchByIdType search );
/**
@@ -140,7 +140,7 @@ public interface GraphManager extends CPManager {
*
* @return An observable that emits Edges. The observer will need to unsubscribe when it has completed consumption.
*/
- Observable<Edge> loadEdgesToTargetByType( SearchByIdType search );
+ Observable<MarkedEdge> loadEdgesToTargetByType( SearchByIdType search );
/**
* Get all edge types to this node. The node provided by search is the target node.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
index da6fedb..88809e2 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
@@ -30,6 +30,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
* An edge. With the additional info of if it is marked for deletion
*
*/
+@JsonDeserialize(as = SimpleMarkedEdge.class)
public interface MarkedEdge extends Edge{
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 1bcb398..93ae753 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -144,12 +144,12 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Edge> writeEdge( final Edge edge ) {
+ public Observable<MarkedEdge> writeEdge( final Edge edge ) {
GraphValidation.validateEdge( edge );
final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, false );
- final Observable<Edge> observable = Observable.just( markedEdge ).map( edge1 -> {
+ final Observable<MarkedEdge> observable = Observable.just( markedEdge ).map( edge1 -> {
final UUID timestamp = UUIDGenerator.newTimeUUID();
@@ -175,12 +175,12 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Edge> markEdge( final Edge edge ) {
+ public Observable<MarkedEdge> markEdge( final Edge edge ) {
GraphValidation.validateEdge( edge );
final MarkedEdge markedEdge = new SimpleMarkedEdge( edge, true );
- final Observable<Edge> observable = Observable.just( markedEdge ).map( edge1 -> {
+ final Observable<MarkedEdge> observable = Observable.just( markedEdge ).map( edge1 -> {
final UUID timestamp = UUIDGenerator.newTimeUUID();
@@ -282,9 +282,9 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Edge> loadEdgeVersions( final SearchByEdge searchByEdge ) {
+ public Observable<MarkedEdge> loadEdgeVersions( final SearchByEdge searchByEdge ) {
- final Observable<Edge> edges =
+ final Observable<MarkedEdge> edges =
Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
@@ -298,8 +298,8 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
- final Observable<Edge> edges =
+ public Observable<MarkedEdge> loadEdgesFromSource( final SearchByEdgeType search ) {
+ final Observable<MarkedEdge> edges =
Observable.create( new ObservableIterator<MarkedEdge>( "loadEdgesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
@@ -313,8 +313,8 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
- final Observable<Edge> edges =
+ public Observable<MarkedEdge> loadEdgesToTarget( final SearchByEdgeType search ) {
+ final Observable<MarkedEdge> edges =
Observable.create( new ObservableIterator<MarkedEdge>( "loadEdgesToTarget" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
@@ -329,8 +329,8 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
- final Observable<Edge> edges =
+ public Observable<MarkedEdge> loadEdgesFromSourceByType( final SearchByIdType search ) {
+ final Observable<MarkedEdge> edges =
Observable.create( new ObservableIterator<MarkedEdge>( "loadEdgesFromSourceByType" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
@@ -344,8 +344,8 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
- final Observable<Edge> edges =
+ public Observable<MarkedEdge> loadEdgesToTargetByType( final SearchByIdType search ) {
+ final Observable<MarkedEdge> edges =
Observable.create( new ObservableIterator<MarkedEdge>( "loadEdgesToTargetByType" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
@@ -480,7 +480,7 @@ public class GraphManagerImpl implements GraphManager {
}
//if any one of these is true, we filter it
- return !(simpleMarkedEdge.isDeleted() || simpleMarkedEdge.isSourceNodeDelete() || simpleMarkedEdge.isTargetNodeDeleted());
+ return !simpleMarkedEdge.isDeleted() && !simpleMarkedEdge.isSourceNodeDelete() && !simpleMarkedEdge.isTargetNodeDeleted();
});
} );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
index c6dc2e4..9c35e2e 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
@@ -29,50 +29,61 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
/**
* Simple bean to represent our edge
+ *
* @author tnine
*/
-public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
+public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
+
+ private boolean isDeleted;
+ private boolean isSourceNodeDeleted;
+ private boolean isTargetNodeDeleted;
- private final boolean deleted;
- private final boolean isSourceNodeDeleted;
- private final boolean isTargetNodeDeleted;
+ /**
+ * Unused but required for Jackson
+ */
+ public SimpleMarkedEdge() {
+ }
- public SimpleMarkedEdge( final Id sourceNode, final String type, final Id targetNode, final long timestamp, final boolean deleted) {
- this( sourceNode, type, targetNode, timestamp, deleted, false, false );
+ public SimpleMarkedEdge( final Id sourceNode, final String type, final Id targetNode, final long timestamp,
+ final boolean isDeleted ) {
+
+ this( sourceNode, type, targetNode, timestamp, isDeleted, false, false );
}
public SimpleMarkedEdge( final Id sourceNode, final String type, final Id targetNode, final long timestamp,
- final boolean deleted, final boolean isSourceNodeDeleted,
+ final boolean isDeleted, final boolean isSourceNodeDeleted,
final boolean isTargetNodeDeleted ) {
super( sourceNode, type, targetNode, timestamp );
- this.deleted = deleted;
+ this.isDeleted = isDeleted;
this.isSourceNodeDeleted = isSourceNodeDeleted;
this.isTargetNodeDeleted = isTargetNodeDeleted;
}
- public SimpleMarkedEdge(final Edge edge, final boolean deleted){
- this(edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(), deleted);
+ public SimpleMarkedEdge( final Edge edge, final boolean isDeleted ) {
+ this( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(), isDeleted );
}
@Override
@JsonIgnore
public boolean isDeleted() {
- return deleted;
+ return isDeleted;
}
@Override
+ @JsonIgnore
public boolean isSourceNodeDelete() {
return isSourceNodeDeleted;
}
@Override
+ @JsonIgnore
public boolean isTargetNodeDeleted() {
return isTargetNodeDeleted;
}
@@ -92,7 +103,7 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
final SimpleMarkedEdge that = ( SimpleMarkedEdge ) o;
- if ( deleted != that.deleted ) {
+ if ( isDeleted != that.isDeleted ) {
return false;
}
@@ -103,7 +114,7 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
@Override
public int hashCode() {
int result = super.hashCode();
- result = 31 * result + ( deleted ? 1 : 0 );
+ result = 31 * result + ( isDeleted ? 1 : 0 );
result = 31 * result + ( isSourceNodeDeleted ? 1 : 0 );
result = 31 * result + ( isTargetNodeDeleted ? 1 : 0 );
return result;
@@ -113,7 +124,7 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
@Override
public String toString() {
return "SimpleMarkedEdge{" +
- "deleted=" + deleted +
+ "deleted=" + isDeleted +
", isSourceNodeDeleted=" + isSourceNodeDeleted +
", isTargetNodeDeleted=" + isTargetNodeDeleted +
"} " + super.toString();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
index eda3a02..9a95c26 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerIT.java
@@ -95,7 +95,7 @@ public class GraphManagerIT {
SearchByEdgeType search = createSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesFromSource( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSource( search );
//implicitly blows up if more than 1 is returned from "single"
Edge returned = edges.toBlocking().last();
@@ -127,7 +127,7 @@ public class GraphManagerIT {
SearchByEdgeType search = createSearchByEdge( edge.getTargetNode(), edge.getType(), edge.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesToTarget( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesToTarget( search );
//implicitly blows up if more than 1 is returned from "single"
Edge returned = edges.toBlocking().single();
@@ -161,7 +161,7 @@ public class GraphManagerIT {
SearchByEdgeType search = createSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesFromSource( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSource( search );
//implicitly blows up if more than 1 is returned from "single"
Edge returned = edges.toBlocking().single();
@@ -196,7 +196,7 @@ public class GraphManagerIT {
SearchByEdgeType search = createSearchByEdge( edge.getTargetNode(), edge.getType(), edge.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesToTarget( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesToTarget( search );
//implicitly blows up if more than 1 is returned from "single"
Edge returned = edges.toBlocking().single();
@@ -248,10 +248,10 @@ public class GraphManagerIT {
SearchByEdgeType search =
createSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge3.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesFromSource( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSource( search );
//implicitly blows up if more than 1 is returned from "single"
- Iterator<Edge> returned = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> returned = edges.toBlocking().getIterator();
assertEquals( "Correct edge returned", edge3, returned.next() );
assertEquals( "Correct edge returned", edge2, returned.next() );
@@ -321,10 +321,10 @@ public class GraphManagerIT {
SearchByEdgeType search =
createSearchByEdge( edge1.getTargetNode(), edge1.getType(), edge3.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesToTarget( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesToTarget( search );
//implicitly blows up if more than 1 is returned from "single"
- Iterator<Edge> returned = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> returned = edges.toBlocking().getIterator();
assertEquals( "Correct edge returned", edge3, returned.next() );
assertEquals( "Correct edge returned", edge2, returned.next() );
@@ -387,10 +387,10 @@ public class GraphManagerIT {
SearchByEdgeType search =
createSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge3.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesFromSource( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSource( search );
//implicitly blows up if more than 1 is returned from "single"
- Iterator<Edge> returned = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> returned = edges.toBlocking().getIterator();
//we have 3 edges, but we specified our first edge as the max, we shouldn't get any more results than the first
@@ -443,10 +443,10 @@ public class GraphManagerIT {
SearchByEdgeType search =
createSearchByEdge( edge1.getTargetNode(), edge1.getType(), edge3.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesToTarget( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesToTarget( search );
//implicitly blows up if more than 1 is returned from "single"
- Iterator<Edge> returned = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> returned = edges.toBlocking().getIterator();
//we have 3 edges, but we specified our first edge as the max, we shouldn't get any more results than the first
@@ -487,7 +487,7 @@ public class GraphManagerIT {
SearchByIdType search = createSearchByEdgeAndId( edge.getSourceNode(), edge.getType(), edge.getTimestamp(),
edge.getTargetNode().getType(), null );
- Observable<Edge> edges = gm.loadEdgesFromSourceByType( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSourceByType( search );
//implicitly blows up if more than 1 is returned from "single"
Edge returned = edges.toBlocking().single();
@@ -524,7 +524,7 @@ public class GraphManagerIT {
SearchByIdType search = createSearchByEdgeAndId( edge.getTargetNode(), edge.getType(), edge.getTimestamp(),
edge.getSourceNode().getType(), null );
- Observable<Edge> edges = gm.loadEdgesToTargetByType( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesToTargetByType( search );
//implicitly blows up if more than 1 is returned from "single"
Edge returned = edges.toBlocking().single();
@@ -560,7 +560,7 @@ public class GraphManagerIT {
SearchByEdgeType search = createSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesFromSource( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSource( search );
//implicitly blows up if more than 1 is returned from "single"
Edge returned = edges.toBlocking().single();
@@ -629,7 +629,7 @@ public class GraphManagerIT {
SearchByEdgeType search = createSearchByEdge( edge.getTargetNode(), edge.getType(), edge.getTimestamp(), null );
- Observable<Edge> edges = gm.loadEdgesToTarget( search );
+ Observable<MarkedEdge> edges = gm.loadEdgesToTarget( search );
//implicitly blows up if more than 1 is returned from "single"
Edge returned = edges.toBlocking().single();
@@ -988,11 +988,11 @@ public class GraphManagerIT {
//get our 2 edges
- Observable<Edge> edges =
+ Observable<MarkedEdge> edges =
gm.loadEdgesFromSource( createSearchByEdge( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
- Iterator<Edge> results = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> results = edges.toBlocking().getIterator();
assertEquals( "Edges correct", edge2, results.next() );
@@ -1060,11 +1060,11 @@ public class GraphManagerIT {
//get our 2 edges
- Observable<Edge> edges =
+ Observable<MarkedEdge> edges =
gm.loadEdgesToTarget( createSearchByEdge( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
- Iterator<Edge> results = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> results = edges.toBlocking().getIterator();
assertEquals( "Edges correct", edge2, results.next() );
@@ -1140,11 +1140,11 @@ public class GraphManagerIT {
//get our 2 edges
- Observable<Edge> edges = gm.loadEdgesFromSource(
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSource(
createSearchByEdgeUnfiltered( edge1.getSourceNode(), edge1.getType(), maxVersion, null ) );
- Iterator<Edge> results = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> results = edges.toBlocking().getIterator();
assertEquals( "Edges correct", edge2.getTargetNode(), results.next().getTargetNode() );
@@ -1211,11 +1211,11 @@ public class GraphManagerIT {
gm.markEdge( edge1 ).toBlocking().last();
//get our 2 edges
- Observable<Edge> edges = gm.loadEdgesToTarget(
+ Observable<MarkedEdge> edges = gm.loadEdgesToTarget(
createSearchByEdgeUnfiltered( edge1.getTargetNode(), edge1.getType(), maxVersion, null ) );
- Iterator<Edge> results = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> results = edges.toBlocking().getIterator();
assertEquals( "Edges correct", edge2.getSourceNode(), results.next().getSourceNode() );
@@ -1282,11 +1282,11 @@ public class GraphManagerIT {
//get our 2 edges
- Observable<Edge> edges = gm.loadEdgesFromSourceByType(
+ Observable<MarkedEdge> edges = gm.loadEdgesFromSourceByType(
createSearchByEdgeAndId( sourceId, edge1.getType(), maxVersion, targetId1.getType(), null ) );
- Iterator<Edge> results = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> results = edges.toBlocking().getIterator();
assertEquals( "Edges correct", edge1, results.next() );
@@ -1359,11 +1359,11 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
//get our 2 edges
- Observable<Edge> edges = gm.loadEdgesToTargetByType(
+ Observable<MarkedEdge> edges = gm.loadEdgesToTargetByType(
createSearchByEdgeAndId( targetId, edge1.getType(), maxVersion, sourceId1.getType(), null ) );
- Iterator<Edge> results = edges.toBlocking().getIterator();
+ Iterator<MarkedEdge> results = edges.toBlocking().getIterator();
assertEquals( "Edges correct", edge1, results.next() );
@@ -1435,7 +1435,7 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
- Iterator<Edge> results =
+ Iterator<MarkedEdge> results =
gm.loadEdgesFromSource( createSearchByEdge( sourceId, edge1.getType(), maxVersion, null ) ).toBlocking()
.getIterator();
@@ -1547,7 +1547,7 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
- Iterator<Edge> results =
+ Iterator<MarkedEdge> results =
gm.loadEdgesFromSource( createSearchByEdge( sourceId, edge1.getType(), maxVersion, null ) ).toBlocking()
.getIterator();
@@ -1627,7 +1627,7 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
- Iterator<Edge> results =
+ Iterator<MarkedEdge> results =
gm.loadEdgesToTarget( createSearchByEdge( targetId, edge1.getType(), maxVersion, null ) ).toBlocking()
.getIterator();
@@ -1764,7 +1764,7 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
- Iterator<Edge> results =
+ Iterator<MarkedEdge> results =
gm.loadEdgesToTarget( createSearchByEdge( targetId, edge1.getType(), maxVersion, null ) ).toBlocking()
.getIterator();
@@ -1842,7 +1842,7 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
- Iterator<Edge> results =
+ Iterator<MarkedEdge> results =
gm.loadEdgesFromSource( createSearchByEdge( sourceId, edge1.getType(), maxVersion, null ) ).toBlocking()
.getIterator();
@@ -1983,7 +1983,7 @@ public class GraphManagerIT {
final long maxVersion = System.currentTimeMillis();
- Iterator<Edge> results =
+ Iterator<MarkedEdge> results =
gm.loadEdgesToTarget( createSearchByEdge( targetId, edge1.getType(), maxVersion, null ) ).toBlocking()
.getIterator();
@@ -2356,10 +2356,10 @@ public class GraphManagerIT {
new SimpleSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge1.getTargetNode(), Long.MAX_VALUE,
SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() );
- final Observable<Edge> edgesDescending = gm.loadEdgeVersions( searchDescending );
+ final Observable<MarkedEdge> edgesDescending = gm.loadEdgeVersions( searchDescending );
//search descending
- final List<Edge> descending = edgesDescending.toList().toBlocking().single();
+ final List<MarkedEdge> descending = edgesDescending.toList().toBlocking().single();
assertEquals( "Correct size returned", 3, descending.size() );
@@ -2376,9 +2376,9 @@ public class GraphManagerIT {
new SimpleSearchByEdge( edge1.getSourceNode(), edge1.getType(), edge1.getTargetNode(), 0,
SearchByEdgeType.Order.ASCENDING, Optional.<Edge>absent() );
- Observable<Edge> edgesAscending = gm.loadEdgeVersions( searchAscending );
+ Observable<MarkedEdge> edgesAscending = gm.loadEdgeVersions( searchAscending );
- List<Edge> ascending = edgesAscending.toList().toBlocking().single();
+ List<MarkedEdge> ascending = edgesAscending.toList().toBlocking().single();
assertEquals( "Correct size returned", 3, ascending.size() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
index b922e7c..22683f6 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
@@ -113,7 +113,7 @@ public class GraphManagerLoadTest {
@Override
- public Observable<Edge> doSearch( final GraphManager manager ) {
+ public Observable<MarkedEdge> doSearch( final GraphManager manager ) {
return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, Optional
.<Edge>absent()) );
}
@@ -141,7 +141,7 @@ public class GraphManagerLoadTest {
@Override
- public Observable<Edge> doSearch( final GraphManager manager ) {
+ public Observable<MarkedEdge> doSearch( final GraphManager manager ) {
return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
};
@@ -220,7 +220,7 @@ public class GraphManagerLoadTest {
final CountDownLatch latch = new CountDownLatch( 1 );
- generator.doSearch( manager ).take( readCount ).buffer( 1000 ).subscribe( new Subscriber<List<Edge>>() {
+ generator.doSearch( manager ).take( readCount ).buffer( 1000 ).subscribe( new Subscriber<List<MarkedEdge>>() {
@Override
public void onCompleted() {
timer.stop();
@@ -235,7 +235,7 @@ public class GraphManagerLoadTest {
@Override
- public void onNext( final List<Edge> edges ) {
+ public void onNext( final List<MarkedEdge> edges ) {
log.info("Read {} edges", edges.size());
}
} );
@@ -263,6 +263,6 @@ public class GraphManagerLoadTest {
* @param manager
* @return
*/
- public Observable<Edge> doSearch( final GraphManager manager );
+ public Observable<MarkedEdge> doSearch( final GraphManager manager );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index c1917bb..6aad289 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -176,7 +176,7 @@ public class GraphManagerShardConsistencyIT {
@Override
- public Observable<Edge> doSearch( final GraphManager manager ) {
+ public Observable<MarkedEdge> doSearch( final GraphManager manager ) {
return manager.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent() ) );
@@ -409,7 +409,7 @@ public class GraphManagerShardConsistencyIT {
@Override
- public Observable<Edge> doSearch( final GraphManager manager ) {
+ public Observable<MarkedEdge> doSearch( final GraphManager manager ) {
return manager.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent(), false ) );
@@ -729,7 +729,7 @@ public class GraphManagerShardConsistencyIT {
/**
* Perform the search returning an observable edge
*/
- public Observable<Edge> doSearch( final GraphManager manager );
+ public Observable<MarkedEdge> doSearch( final GraphManager manager );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
index 2889684..6a2efc9 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
@@ -107,25 +107,25 @@ public class GraphManagerStressTest {
@Override
- public Observable<Edge> doSearch( final GraphManager manager ) {
+ public Observable<MarkedEdge> doSearch( final GraphManager manager ) {
final long timestamp = System.currentTimeMillis();
- return Observable.create( new Observable.OnSubscribe<Edge>() {
+ return Observable.create( new Observable.OnSubscribe<MarkedEdge>() {
@Override
- public void call( final Subscriber<? super Edge> subscriber ) {
+ public void call( final Subscriber<? super MarkedEdge> subscriber ) {
try {
for ( Id sourceId : sourceIds ) {
- final Iterable<Edge> edges = manager.loadEdgesFromSource(
+ final Iterable<MarkedEdge> edges = manager.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceId, "test", timestamp, SearchByEdgeType.Order.DESCENDING, Optional
.<Edge>absent() ) )
.toBlocking().toIterable();
- for ( Edge edge : edges ) {
+ for ( MarkedEdge edge : edges ) {
log.debug( "Firing on next for edge {}", edge );
subscriber.onNext( edge );
@@ -195,7 +195,7 @@ public class GraphManagerStressTest {
@Override
- public Observable<Edge> doSearch( final GraphManager manager ) {
+ public Observable<MarkedEdge> doSearch( final GraphManager manager ) {
return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
};
@@ -222,7 +222,7 @@ public class GraphManagerStressTest {
@Override
- public Observable<Edge> doSearch( final GraphManager manager ) {
+ public Observable<MarkedEdge> doSearch( final GraphManager manager ) {
return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
@@ -307,6 +307,6 @@ public class GraphManagerStressTest {
*/
public Edge newEdge();
- public Observable<Edge> doSearch( final GraphManager manager );
+ public Observable<MarkedEdge> doSearch( final GraphManager manager );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d4f22c5/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java b/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
index d9d3d0d..6e84601 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/AppInfoMigrationPlugin.java
@@ -47,6 +47,7 @@ import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsExcept
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
@@ -239,7 +240,7 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
final EntityCollectionManager managementCollectionManager =
entityCollectionManagerFactory.createCollectionManager( managementAppScope );
- Observable<Edge> edgesObservable = getApplicationInfoEdges( appId );
+ Observable<MarkedEdge> edgesObservable = getApplicationInfoEdges( appId );
//get the graph for all app infos
Observable<org.apache.usergrid.persistence.model.entity.Entity> entityObs = edgesObservable.flatMap( edge -> {
final Id appInfoId = edge.getTargetNode();
@@ -299,7 +300,7 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
}
- public Observable<Edge> getApplicationInfoEdges( final UUID applicationId ) {
+ public Observable<MarkedEdge> getApplicationInfoEdges( final UUID applicationId ) {
final ApplicationScope managementAppScope = getApplicationScope( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
final GraphManager gm = graphManagerFactory.createEdgeManager( managementAppScope );
[27/50] [abbrv] usergrid git commit: Change read repair to interact
with c* directly and only fire and index operation message to get the ES
document removed from all regions.
Posted by to...@apache.org.
Change read repair to interact with c* directly and only fire and index operation message to get the ES document removed from all regions.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b43bda3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b43bda3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b43bda3
Branch: refs/heads/master
Commit: 1b43bda3f801172b0e59f927ff3ae52a559d36cc
Parents: 70d7a95
Author: Michael Russo <mi...@gmail.com>
Authored: Tue Oct 27 10:50:16 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Tue Oct 27 12:56:13 2015 -0700
----------------------------------------------------------------------
.../asyncevents/AsyncEventService.java | 5 ++
.../asyncevents/InMemoryAsyncEventService.java | 5 ++
.../read/traverse/AbstractReadGraphFilter.java | 69 +++++++++++++++++---
.../traverse/ReadGraphCollectionFilter.java | 10 ++-
.../traverse/ReadGraphConnectionFilter.java | 10 ++-
.../impl/stage/NodeDeleteListenerImpl.java | 27 ++++----
6 files changed, 97 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index dcfffcb..dbf8996 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -77,6 +77,11 @@ public interface AsyncEventService extends ReIndexAction {
*/
void queueEntityDelete(final ApplicationScope applicationScope, final Id entityId);
+ /**
+ *
+ * @param indexOperationMessage
+ */
+ void queueIndexOperationMessage( final IndexOperationMessage indexOperationMessage );
/**
* current queue depth
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
index fc6385c..d8334b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
@@ -105,6 +105,11 @@ public class InMemoryAsyncEventService implements AsyncEventService {
run( results.getCompactedNode() );
}
+ @Override
+ public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage){
+ //this is not used locally
+ }
+
public void index( final ApplicationScope applicationScope, final Id id, final long updatedSince ) {
final EntityIndexOperation entityIndexOperation = new EntityIndexOperation( applicationScope, id, updatedSince );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 9d050c8..89230d7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,7 +20,11 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,6 +46,7 @@ import com.google.common.base.Optional;
import rx.Observable;
+
/**
* Command for reading graph edges
*/
@@ -51,15 +56,21 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
private final GraphManagerFactory graphManagerFactory;
private final RxTaskScheduler rxTaskScheduler;
+ private final EventBuilder eventBuilder;
+ private final AsyncEventService asyncEventService;
/**
* Create a new instance of our command
*/
public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory,
- final RxTaskScheduler rxTaskScheduler) {
+ final RxTaskScheduler rxTaskScheduler,
+ final EventBuilder eventBuilder,
+ final AsyncEventService asyncEventService ) {
this.graphManagerFactory = graphManagerFactory;
this.rxTaskScheduler = rxTaskScheduler;
+ this.eventBuilder = eventBuilder;
+ this.asyncEventService = asyncEventService;
}
@@ -107,28 +118,56 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
final boolean isTargetNodeDelete = markedEdge.isTargetNodeDeleted();
+ if (isDeleted) {
- if(isDeleted){
logger.trace("Edge {} is deleted, deleting the edge", markedEdge);
- graphManager.deleteEdge(markedEdge).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
+
+ indexMessageObservable
+ .compose(applyCollector())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
+
}
- if(isSourceNodeDeleted){
- final Id sourceNodeId = markedEdge.getSourceNode();
+ if (isSourceNodeDeleted) {
+ final Id sourceNodeId = markedEdge.getSourceNode();
logger.trace("Edge {} has a deleted source node, deleting the entity for id {}", markedEdge, sourceNodeId);
- graphManager.compactNode(sourceNodeId).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+
+ final EventBuilderImpl.EntityDeleteResults
+ entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
+
+ entityDeleteResults.getIndexObservable()
+ .compose(applyCollector())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
+
+ Observable.merge(entityDeleteResults.getEntitiesDeleted(),
+ entityDeleteResults.getCompactedNode())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
+ subscribe();
+
}
- if(isTargetNodeDelete){
+ if (isTargetNodeDelete) {
final Id targetNodeId = markedEdge.getTargetNode();
+ logger.trace("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId);
- logger.trace("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId );
- graphManager.compactNode(targetNodeId).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ final EventBuilderImpl.EntityDeleteResults
+ entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
+
+ entityDeleteResults.getIndexObservable()
+ .compose(applyCollector())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
+
+ Observable.merge(entityDeleteResults.getEntitiesDeleted(),
+ entityDeleteResults.getCompactedNode())
+ .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
+ subscribe();
+
}
@@ -202,4 +241,16 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
return cursorEdge;
}
}
+
+ private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector() {
+
+ return observable -> observable
+ .filter((IndexOperationMessage msg) -> !msg.isEmpty())
+ .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
+ .doOnNext(indexOperation -> {
+ asyncEventService.queueIndexOperationMessage(indexOperation);
+ });
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
index 1d63bc6..3d7df3b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
@@ -41,8 +43,12 @@ public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, final RxTaskScheduler rxTaskScheduler, @Assisted final String collectionName ) {
- super( graphManagerFactory, rxTaskScheduler );
+ public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory,
+ final RxTaskScheduler rxTaskScheduler,
+ final EventBuilder eventBuilder,
+ final AsyncEventService asyncEventService,
+ @Assisted final String collectionName ) {
+ super( graphManagerFactory, rxTaskScheduler, eventBuilder, asyncEventService );
this.collectionName = collectionName;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
index efe94db..b2d368b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
@@ -41,8 +43,12 @@ public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, final RxTaskScheduler rxTaskScheduler, @Assisted final String connectionName ) {
- super( graphManagerFactory, rxTaskScheduler );
+ public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory,
+ final RxTaskScheduler rxTaskScheduler,
+ final EventBuilder eventBuilder,
+ final AsyncEventService asyncEventService,
+ @Assisted final String connectionName ) {
+ super( graphManagerFactory, rxTaskScheduler, eventBuilder, asyncEventService );
this.connectionName = connectionName;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b43bda3/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index e4eb5fc..343cc77 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@ -157,26 +157,26 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
//get all edges pointing to the target node and buffer then into groups for deletion
Observable<MarkedEdge> targetEdges =
- getEdgesTypesToTarget( scope, new SimpleSearchEdgeType( node, null, null ) )
- .subscribeOn( Schedulers.io() ).flatMap( edgeType -> Observable.create( new ObservableIterator<MarkedEdge>( "getTargetEdges" ) {
+ getEdgesTypesToTarget(scope, new SimpleSearchEdgeType(node, null, null))
+ .flatMap(edgeType -> Observable.create(new ObservableIterator<MarkedEdge>("getTargetEdges") {
@Override
protected Iterator<MarkedEdge> getIterator() {
- return storageSerialization.getEdgesToTarget( scope,
- new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
+ return storageSerialization.getEdgesToTarget(scope,
+ new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
}
- } ) );
+ }));
//get all edges pointing to the source node and buffer them into groups for deletion
Observable<MarkedEdge> sourceEdges =
- getEdgesTypesFromSource( scope, new SimpleSearchEdgeType( node, null, null ) )
- .subscribeOn( Schedulers.io() ).flatMap( edgeType -> Observable.create( new ObservableIterator<MarkedEdge>( "getSourceEdges" ) {
+ getEdgesTypesFromSource(scope, new SimpleSearchEdgeType(node, null, null))
+ .flatMap(edgeType -> Observable.create(new ObservableIterator<MarkedEdge>("getSourceEdges") {
@Override
protected Iterator<MarkedEdge> getIterator() {
- return storageSerialization.getEdgesFromSource( scope,
- new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
+ return storageSerialization.getEdgesFromSource(scope,
+ new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
}
- } ) );
+ }));
//merge both source and target into 1 observable. We'll need to check them all regardless of order
return Observable.merge( targetEdges, sourceEdges )
@@ -235,12 +235,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
//run both the source/target edge type cleanup, then proceed
return Observable.merge( sourceMetaCleanup, targetMetaCleanup ).lastOrDefault( null )
- .flatMap( new Func1<Integer, Observable<MarkedEdge>>() {
- @Override
- public Observable<MarkedEdge> call( final Integer integer ) {
- return Observable.from( markedEdges );
- }
- } );
+ .flatMap(integer -> Observable.from( markedEdges ));
} );
}
[30/50] [abbrv] usergrid git commit: -Convert search in ES to use
term queries instead of scroll with filters. -Fix delete so we remove
connections from Elasticsearch as well as the entity on entity delete event.
Posted by to...@apache.org.
-Convert search in ES to use term queries instead of scroll with filters.
-Fix delete so we remove connections from Elasticsearch as well as the entity on entity delete event.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/22beca2c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/22beca2c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/22beca2c
Branch: refs/heads/master
Commit: 22beca2cb3714833ddf6a748522c6bead17fe9f2
Parents: 2686054
Author: Michael Russo <mi...@gmail.com>
Authored: Wed Oct 28 15:58:41 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Wed Oct 28 15:58:41 2015 -0700
----------------------------------------------------------------------
.../corepersistence/index/IndexServiceImpl.java | 2 +-
.../persistence/index/CandidateResult.java | 11 +-
.../persistence/index/EntityIndexBatch.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 6 +
.../index/impl/DeIndexOperation.java | 5 +
.../index/impl/EsEntityIndexBatchImpl.java | 34 ++++-
.../index/impl/EsEntityIndexImpl.java | 149 +++++++++++--------
.../persistence/index/impl/IndexingUtils.java | 2 +-
8 files changed, 137 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index ccb6221..b2a1a2a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -219,7 +219,7 @@ public class IndexServiceImpl implements IndexService {
//collect results into a single batch
.collect( () -> ei.createBatch(), ( batch, candidateResult ) -> {
logger.debug( "Deindexing on edge {} for entity {} added to batch",searchEdge , entityId );
- batch.deindex( searchEdge, candidateResult );
+ batch.deindex( candidateResult );
} )
//return the future from the batch execution
.flatMap( batch ->Observable.just(batch.build()) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java
index 2375509..9b1898c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/CandidateResult.java
@@ -32,10 +32,12 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class CandidateResult implements EntityVersion {
private final Id entityId;
private final UUID entityVersion;
+ private final String docId;
- public CandidateResult( Id entityId, UUID entityVersion ) {
+ public CandidateResult( Id entityId, UUID entityVersion, String docId ) {
this.entityId = entityId;
this.entityVersion = entityVersion;
+ this.docId = docId;
}
@Override
@@ -48,6 +50,10 @@ public class CandidateResult implements EntityVersion {
return entityId;
}
+ public String getDocId() {
+ return docId;
+ }
+
@Override
public boolean equals( final Object o ) {
@@ -66,6 +72,9 @@ public class CandidateResult implements EntityVersion {
if ( !entityVersion.equals( that.entityVersion ) ) {
return false;
}
+ if ( !docId.equals( that.docId ) ) {
+ return false;
+ }
return true;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index 98652c1..17dd4d3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -63,7 +63,7 @@ public interface EntityIndexBatch {
EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID version );
-
+ EntityIndexBatch deindex( final CandidateResult candidateResult);
/**
* get the batches
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index e093d7d..33f199a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -60,6 +60,8 @@ public interface IndexFig extends GuicyFig {
String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
+ String ELASTICSEARCH_VERSION_QUERY_LIMIT = "elasticsearch.version_query_limit";
+
/**
* Comma-separated list of Elasticsearch hosts.
@@ -199,4 +201,8 @@ public interface IndexFig extends GuicyFig {
@Default("1000")
@Key( "elasticsearch_queue_error_sleep_ms" )
long getSleepTimeForQueueError();
+
+ @Default("1000")
+ @Key( ELASTICSEARCH_VERSION_QUERY_LIMIT )
+ int getVersionQueryLimit();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
index 4060dac..dbecf8a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/DeIndexOperation.java
@@ -61,6 +61,11 @@ public class DeIndexOperation implements BatchOperation {
this.documentId = createIndexDocId( applicationScope, id, version, searchEdge );
}
+ public DeIndexOperation( String[] indexes, String docId) {
+ this.indexes = indexes;
+ this.documentId = docId;
+ }
+
@Override
public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 68830ca..a8fb751 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -84,23 +84,42 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID version ) {
- IndexValidationUtils.validateSearchEdge( searchEdge );
- ValidationUtils.verifyIdentity( id );
+ IndexValidationUtils.validateSearchEdge(searchEdge);
+ ValidationUtils.verifyIdentity(id);
ValidationUtils.verifyVersion( version );
String[] indexes = entityIndex.getIndexes();
//get the default index if no alias exists yet
if ( indexes == null || indexes.length == 0 ) {
- throw new IllegalStateException("No indexes exist for " + indexLocationStrategy.getAlias().getWriteAlias());
+ throw new IllegalStateException("No indexes exist for " + indexLocationStrategy.getAlias().getWriteAlias());
}
if ( log.isDebugEnabled() ) {
log.debug( "Deindexing to indexes {} with scope {} on edge {} with id {} and version {} ",
- new Object[] { indexes, applicationScope, searchEdge, id, version } );
+ new Object[] { indexes, applicationScope, searchEdge, id, version } );
}
- container.addDeIndexRequest( new DeIndexOperation( indexes, applicationScope, searchEdge, id, version ) );
+ container.addDeIndexRequest(new DeIndexOperation(indexes, applicationScope, searchEdge, id, version));
+
+ return this;
+ }
+
+ public EntityIndexBatch deindexWithDocId( final String docId ) {
+
+ String[] indexes = entityIndex.getIndexes();
+ //get the default index if no alias exists yet
+ if ( indexes == null || indexes.length == 0 ) {
+ throw new IllegalStateException("No indexes exist for " + indexLocationStrategy.getAlias().getWriteAlias());
+ }
+
+ if ( log.isDebugEnabled() ) {
+ log.debug( "Deindexing to indexes {} with with documentId {} ",
+ new Object[] { indexes, docId } );
+ }
+
+
+ container.addDeIndexRequest( new DeIndexOperation( indexes, docId ) );
return this;
}
@@ -117,6 +136,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
return deindex( searchEdge, entity.getId(), entity.getVersion() );
}
+ @Override
+ public EntityIndexBatch deindex( final CandidateResult entity ) {
+
+ return deindexWithDocId(entity.getDocId());
+ }
@Override
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 9be591e..0b7f2d7 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -65,6 +65,7 @@ import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
+import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -389,7 +390,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final ParsedQuery parsedQuery = ParsedQueryBuilder.build(query);
final SearchRequestBuilder srb = searchRequest.getBuilder( searchEdge, searchTypes, parsedQuery, limit, offset )
- .setTimeout( TimeValue.timeValueMillis( queryTimeout ) );
+ .setTimeout(TimeValue.timeValueMillis(queryTimeout));
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index (read alias): {}\n nodeId: {}, edgeType: {}, \n type: {}\n query: {} ",
@@ -427,57 +428,62 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
*/
IndexValidationUtils.validateSearchEdge(edge);
- Preconditions.checkNotNull( entityId, "entityId cannot be null" );
+ Preconditions.checkNotNull(entityId, "entityId cannot be null");
SearchResponse searchResponse;
-
List<CandidateResult> candidates = new ArrayList<>();
- final ParsedQuery parsedQuery = ParsedQueryBuilder.build( "select *" );
-
- final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
-
- //I can't just search on the entity Id.
+ //never let the limit be less than 2 as there are potential indefinite paging issues
+ final int searchLimit = Math.max(2, indexFig.getVersionQueryLimit());
- FilterBuilder entityEdgeFilter = FilterBuilders.termFilter(IndexingUtils.EDGE_NODE_ID_FIELDNAME,
- IndexingUtils.nodeId(edge.getNodeId()));
+ final QueryBuilder entityQuery = QueryBuilders
+ .termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(edge.getNodeId()));
- srb.setPostFilter(entityEdgeFilter);
+ final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching for marked versions in index (read alias): {}\n nodeId: {},\n query: {} ",
+ logger.debug( "Searching for edges in (read alias): {}\n nodeId: {},\n query: {} ",
this.alias.getReadAlias(),entityId, srb );
}
try {
- //Added For Graphite Metrics
-
- //set the timeout on the scroll cursor to 6 seconds and set the number of values returned per shard to 100.
- //The settings for the scroll aren't tested and so we aren't sure what vlaues would be best in a production enviroment
- //TODO: review this and make them not magic numbers when acking this PR.
- searchResponse = srb.setScroll( new TimeValue( 6000 ) ).setSize( 100 ).execute().actionGet();
+ long queryTimestamp = 0L;
while(true){
- //add search result hits to some sort of running tally of hits.
- candidates = aggregateScrollResults( candidates, searchResponse, null );
- SearchScrollRequestBuilder ssrb = searchRequestBuilderStrategyV2
- .getScrollBuilder( searchResponse.getScrollId() )
- .setScroll( new TimeValue( 6000 ) );
+ QueryBuilder timestampQuery = QueryBuilders
+ .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
+ .gte(queryTimestamp);
- //TODO: figure out how to log exactly what we're putting into elasticsearch
- // if ( logger.isDebugEnabled() ) {
- // logger.debug( "Scroll search using query: {} ",
- // ssrb.toString() );
- // }
+ QueryBuilder finalQuery = QueryBuilders
+ .boolQuery()
+ .must(entityQuery)
+ .must(timestampQuery);
- searchResponse = ssrb.execute().actionGet();
+ searchResponse = srb
+ .setQuery(finalQuery)
+ .setSize(searchLimit)
+ .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC)
+ .execute()
+ .actionGet();
- if (searchResponse.getHits().getHits().length == 0) {
+ int responseSize = searchResponse.getHits().getHits().length;
+ if(responseSize == 0){
break;
}
+ // update queryTimestamp to be the timestamp of the last entity returned from the query
+ queryTimestamp = (long) searchResponse
+ .getHits().getAt(responseSize - 1)
+ .getSource().get(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME);
+
+ candidates = aggregateScrollResults(candidates, searchResponse, null);
+
+ if(responseSize < searchLimit){
+
+ break;
+ }
}
}
@@ -488,7 +494,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
failureMonitor.success();
- return new CandidateResults( candidates, parsedQuery.getSelectFieldMappings());
+ return new CandidateResults( candidates, Collections.EMPTY_SET);
}
@@ -496,61 +502,74 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
public CandidateResults getAllEntityVersionsBeforeMarkedVersion( final Id entityId, final UUID markedVersion ) {
Preconditions.checkNotNull( entityId, "entityId cannot be null" );
- Preconditions.checkNotNull( markedVersion, "markedVersion cannot be null" );
+ Preconditions.checkNotNull(markedVersion, "markedVersion cannot be null");
ValidationUtils.verifyVersion(markedVersion);
SearchResponse searchResponse;
-
List<CandidateResult> candidates = new ArrayList<>();
- final ParsedQuery parsedQuery = ParsedQueryBuilder.build( "select *" );
+ final long markedTimestamp = markedVersion.timestamp();
- final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
-
- FilterBuilder entityIdFilter = FilterBuilders.termFilter(IndexingUtils.ENTITY_ID_FIELDNAME,
- IndexingUtils.entityId(entityId));
+ // never let the limit be less than 2 as there are potential indefinite paging issues
+ final int searchLimit = Math.max(2, indexFig.getVersionQueryLimit());
- srb.setPostFilter(entityIdFilter);
+ // this query will find the document for the entity itself
+ final QueryBuilder entityQuery = QueryBuilders
+ .termQuery(IndexingUtils.ENTITY_ID_FIELDNAME, IndexingUtils.entityId(entityId));
+ // this query will find all the documents where this entity is a source/target node
+ final QueryBuilder nodeQuery = QueryBuilders
+ .termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(entityId));
-
- if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching for marked versions in index (read alias): {}\n nodeId: {},\n query: {} ",
- this.alias.getReadAlias(),entityId, srb );
- }
+ final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
try {
- //Added For Graphite Metrics
- //set the timeout on the scroll cursor to 6 seconds and set the number of values returned per shard to 100.
- //The settings for the scroll aren't tested and so we aren't sure what vlaues would be best in a production enviroment
- //TODO: review this and make them not magic numbers when acking this PR.
- searchResponse = srb.setScroll( new TimeValue( 6000 ) ).setSize( 100 ).execute().actionGet();
+ long queryTimestamp = 0L;
- //list that will hold all of the search hits
+ while(true){
+ QueryBuilder timestampQuery = QueryBuilders
+ .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
+ .gte(queryTimestamp)
+ .lte(markedTimestamp);
+
+ QueryBuilder entityQueryWithTimestamp = QueryBuilders
+ .boolQuery()
+ .must(entityQuery)
+ .must(timestampQuery);
+
+ QueryBuilder finalQuery = QueryBuilders
+ .boolQuery()
+ .should(entityQueryWithTimestamp)
+ .should(nodeQuery)
+ .minimumNumberShouldMatch(1);
+
+ searchResponse = srb
+ .setQuery(finalQuery)
+ .setSize(searchLimit)
+ .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC)
+ .execute()
+ .actionGet();
- while(true){
- //add search result hits to some sort of running tally of hits.
- candidates = aggregateScrollResults( candidates, searchResponse, markedVersion);
+ int responseSize = searchResponse.getHits().getHits().length;
+ if(responseSize == 0){
+ break;
+ }
+
+ candidates = aggregateScrollResults(candidates, searchResponse, markedVersion);
- SearchScrollRequestBuilder ssrb = searchRequestBuilderStrategyV2
- .getScrollBuilder( searchResponse.getScrollId() )
- .setScroll( new TimeValue( 6000 ) );
+ // update queryTimestamp to be the timestamp of the last entity returned from the query
+ queryTimestamp = (long) searchResponse
+ .getHits().getAt(responseSize - 1)
+ .getSource().get(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME);
- //TODO: figure out how to log exactly what we're putting into elasticsearch
-// if ( logger.isDebugEnabled() ) {
-// logger.debug( "Scroll search using query: {} ",
-// ssrb.toString() );
-// }
- searchResponse = ssrb.execute().actionGet();
+ if(responseSize < searchLimit){
- if (searchResponse.getHits().getHits().length == 0) {
break;
}
-
}
}
catch ( Throwable t ) {
@@ -560,7 +579,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
failureMonitor.success();
- return new CandidateResults( candidates, parsedQuery.getSelectFieldMappings());
+ return new CandidateResults( candidates, Collections.EMPTY_SET);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22beca2c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 9e06fa6..18cb928 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -241,7 +241,7 @@ public class IndexingUtils {
Id entityId = new SimpleId( UUID.fromString( entityUUID ), entityType );
- return new CandidateResult( entityId, UUID.fromString( versionUUID ) );
+ return new CandidateResult( entityId, UUID.fromString( versionUUID ), documentId );
}
[10/50] [abbrv] usergrid git commit: Added back pressure block as a
test
Posted by to...@apache.org.
Added back pressure block as a test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f73ac4c7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f73ac4c7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f73ac4c7
Branch: refs/heads/master
Commit: f73ac4c724088150bc1e0315942e93306321a72a
Parents: d35fea5
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 23 10:45:10 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 23 10:50:27 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/EventBuilderImpl.java | 3 ++-
.../usergrid/corepersistence/pipeline/read/FilterResult.java | 7 +++++++
.../pipeline/read/traverse/AbstractReadGraphFilter.java | 2 +-
.../usergrid/persistence/graph/impl/GraphManagerImpl.java | 8 ++++----
4 files changed, 14 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f73ac4c7/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index cc0356b..18f080b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -155,7 +155,8 @@ public class EventBuilderImpl implements EventBuilder {
//observable of entries as the batches are deleted
final Observable<List<MvccLogEntry>> entries =
ecm.getVersions( entityId ).buffer( serializationFig.getBufferSize() )
- .doOnNext( buffer -> ecm.delete( buffer ) ).doOnCompleted( () -> gm.compactNode( entityId ) );
+ .doOnNext( buffer -> ecm.delete( buffer ) ).doOnCompleted( () -> gm.compactNode( entityId ).toBlocking().lastOrDefault(null) );
+
return new EntityDeleteResults( edgeObservable, entries );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f73ac4c7/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterResult.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterResult.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterResult.java
index 3c41a2b..915af03 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterResult.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterResult.java
@@ -53,4 +53,11 @@ public class FilterResult<T> {
}
+ @Override
+ public String toString() {
+ return "FilterResult{" +
+ "path=" + path +
+ ", value=" + value +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f73ac4c7/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index d3e0345..88c912a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -85,7 +85,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
/**
* TODO, pass a message with pointers to our cursor values to be generated later
*/
- return graphManager.loadEdgesFromSource( search )
+ return graphManager.loadEdgesFromSource( search ).onBackpressureBlock()
//set the edge state for cursors
.doOnNext( edge -> {
logger.trace( "Seeking over edge {}", edge );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f73ac4c7/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index e119c59..c1e9cea 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -300,7 +300,7 @@ public class GraphManagerImpl implements GraphManager {
@Override
public Observable<Edge> loadEdgesFromSource( final SearchByEdgeType search ) {
final Observable<Edge> edges =
- Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ Observable.create( new ObservableIterator<MarkedEdge>( "loadEdgesFromSource" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSource( scope, search );
@@ -315,7 +315,7 @@ public class GraphManagerImpl implements GraphManager {
@Override
public Observable<Edge> loadEdgesToTarget( final SearchByEdgeType search ) {
final Observable<Edge> edges =
- Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ Observable.create( new ObservableIterator<MarkedEdge>( "loadEdgesToTarget" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTarget( scope, search );
@@ -331,7 +331,7 @@ public class GraphManagerImpl implements GraphManager {
@Override
public Observable<Edge> loadEdgesFromSourceByType( final SearchByIdType search ) {
final Observable<Edge> edges =
- Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ Observable.create( new ObservableIterator<MarkedEdge>( "loadEdgesFromSourceByType" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
@@ -346,7 +346,7 @@ public class GraphManagerImpl implements GraphManager {
@Override
public Observable<Edge> loadEdgesToTargetByType( final SearchByIdType search ) {
final Observable<Edge> edges =
- Observable.create( new ObservableIterator<MarkedEdge>( "getEdgeTypesFromSource" ) {
+ Observable.create( new ObservableIterator<MarkedEdge>( "loadEdgesToTargetByType" ) {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
[37/50] [abbrv] usergrid git commit: Fixes GET + PUT for credentials
info
Posted by to...@apache.org.
Fixes GET + PUT for credentials info
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4784b34f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4784b34f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4784b34f
Branch: refs/heads/master
Commit: 4784b34f3342b32639ee23beb9ccafa9a5443ed0
Parents: 7fc7c55
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 29 12:37:31 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 14:42:41 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/CredentialsInfo.java | 46 ++++++++++++++++++++
.../rest/applications/users/UserResource.java | 11 +++--
2 files changed, 53 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4784b34f/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java b/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java
index 957f8a8..c7c4cba 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/CredentialsInfo.java
@@ -20,7 +20,10 @@ package org.apache.usergrid.persistence;
import com.fasterxml.jackson.annotation.JsonAnyGetter;
import com.fasterxml.jackson.annotation.JsonAnySetter;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.google.common.base.Preconditions;
+
import java.io.Serializable;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import javax.xml.bind.annotation.XmlRootElement;
@@ -118,6 +121,11 @@ public class CredentialsInfo implements Comparable<CredentialsInfo>,Serializable
}
+ public void setCreated( final Long created ) {
+ this.created = created;
+ }
+
+
@JsonAnySetter
public void setProperty( String key, Object value ) {
properties.put( key, value );
@@ -172,4 +180,42 @@ public class CredentialsInfo implements Comparable<CredentialsInfo>,Serializable
}
return o.created.compareTo( created );
}
+
+
+ /**
+ * Parse the json representation into an object
+ * @param json
+ * @return
+ */
+ public static CredentialsInfo fromJson(final Map<String, Object> json){
+ final boolean recoverable = ( boolean ) json.get( "recoverable");
+
+ final boolean encrypted = ( boolean ) json.get("encrypted");
+ final String hashType = ( String ) json.get( "hashType");
+ final long created = ( long ) json.get( "created");
+ final String secret = ( String ) json.get( "secret" );
+
+ final List<String> cryptoChain = ( List<String> ) json.get( "cryptoChain");
+
+ Preconditions.checkNotNull(created, "created is required");
+ Preconditions.checkNotNull(secret, "secret is required");
+ Preconditions.checkNotNull(cryptoChain, "cryptoChain is required");
+
+ Preconditions.checkArgument(cryptoChain.size() >= 1, "cryptoChain must have 1 or more entries");
+
+
+ final String[] cryptoString = new String[cryptoChain.size()];
+
+ cryptoChain.toArray( cryptoString );
+
+
+ final CredentialsInfo credentialsInfo = new CredentialsInfo();
+ credentialsInfo.setEncrypted( encrypted );
+ credentialsInfo.setHashType( hashType );
+ credentialsInfo.setCreated( created );
+ credentialsInfo.setSecret( secret );
+ credentialsInfo.setCryptoChain( cryptoString );
+
+ return credentialsInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4784b34f/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index 716f367..1116469 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -170,7 +170,7 @@ public class UserResource extends ServiceResource {
@GET
@RequireSystemAccess
@Path("credentials")
- public JSONWithPadding getUserPassword(@QueryParam("callback") @DefaultValue("callback") String callback )
+ public JSONWithPadding getUserCredentials(@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
logger.info( "UserResource.setUserPassword" );
@@ -204,6 +204,7 @@ public class UserResource extends ServiceResource {
@PUT
+ @RequireSystemAccess
@Path("credentials")
public JSONWithPadding setUserCredentials( @Context UriInfo ui, Map<String, Object> json,
@QueryParam("callback") @DefaultValue("callback") String callback )
@@ -217,13 +218,15 @@ public class UserResource extends ServiceResource {
ApiResponse response = createApiResponse();
response.setAction( "set user credentials" );
- Object credentials = json.get( "credentials" );
+ Map<String, Object> credentialsJson = ( Map<String, Object> ) json.get( "credentials" );
- if ( credentials == null ) {
+ if ( credentialsJson == null ) {
throw new IllegalArgumentException( "credentials sub object is required" );
}
+ final CredentialsInfo credentials = CredentialsInfo.fromJson( credentialsJson );
+
UUID applicationId = getApplicationId();
UUID targetUserId = getUserUuid();
@@ -233,7 +236,7 @@ public class UserResource extends ServiceResource {
}
- management.setAppUserCredentialsInfo( applicationId, targetUserId, ( CredentialsInfo ) credentials );
+ management.setAppUserCredentialsInfo( applicationId, targetUserId, credentials );
return new JSONWithPadding( response, callback );
[03/50] [abbrv] usergrid git commit: Change new endpoint to only
require org access.
Posted by to...@apache.org.
Change new endpoint to only require org access.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b83a864
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b83a864
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b83a864
Branch: refs/heads/master
Commit: 1b83a864f97e0361266ebbe21e17c3a146bf46b4
Parents: c9bf87c
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Oct 22 10:50:35 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Oct 22 10:50:35 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/rest/applications/ApplicationResource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b83a864/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
index 4eaf09a..162565f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
@@ -606,7 +606,7 @@ public class ApplicationResource extends ServiceResource {
// Specifically require superuser access as this is setting app properties directly (only way to currently do this
// with Apigee's apigeeMobileConfig
- @RequireSystemAccess
+ @RequireOrganizationAccess
@POST
@Path("apm/apigeeMobileConfig")
@Consumes(APPLICATION_JSON)
[49/50] [abbrv] usergrid git commit: Merge branch '2.1-release' of
https://git-wip-us.apache.org/repos/asf/usergrid
Posted by to...@apache.org.
Merge branch '2.1-release' of https://git-wip-us.apache.org/repos/asf/usergrid
# Conflicts:
# stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
# stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
# stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
# stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
# stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
# stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
# stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b422364e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b422364e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b422364e
Branch: refs/heads/master
Commit: b422364e20c14a657de076b793bc6b7c1160a556
Parents: 9d3cf5b eb2756b
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 30 16:32:23 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 30 16:32:23 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 7 +-
.../corepersistence/CpRelationManager.java | 16 +-
.../asyncevents/AmazonAsyncEventService.java | 99 +--
.../asyncevents/AsyncEventService.java | 5 +
.../asyncevents/EventBuilder.java | 25 +-
.../asyncevents/EventBuilderImpl.java | 60 +-
.../index/IndexProcessorFig.java | 15 +-
.../corepersistence/index/IndexServiceImpl.java | 5 +-
.../pipeline/read/FilterResult.java | 7 +
.../read/traverse/AbstractReadGraphFilter.java | 136 +++-
.../read/traverse/EdgeCursorSerializer.java | 8 +-
.../read/traverse/EntityLoadVerifyFilter.java | 24 +-
.../traverse/ReadGraphCollectionFilter.java | 11 +-
.../ReadGraphConnectionByTypeFilter.java | 11 +-
.../traverse/ReadGraphConnectionFilter.java | 11 +-
.../results/ObservableQueryExecutor.java | 7 +
.../service/StatusServiceImpl.java | 3 +-
.../usergrid/persistence/CredentialsInfo.java | 46 ++
.../persistence/ObservableIterator.java | 83 ---
.../index/AsyncIndexServiceTest.java | 3 +-
.../corepersistence/index/IndexServiceTest.java | 12 +-
.../pipeline/cursor/CursorTest.java | 24 +-
.../service/ConnectionServiceImplTest.java | 5 +-
.../persistence/ApplicationServiceIT.java | 4 +-
.../persistence/core/astyanax/CassandraFig.java | 2 +-
.../core/astyanax/MultiRowColumnIterator.java | 46 +-
.../persistence/core/rx/ObservableIterator.java | 2 +-
.../usergrid/persistence/graph/GraphFig.java | 4 +-
.../persistence/graph/GraphManager.java | 14 +-
.../usergrid/persistence/graph/MarkedEdge.java | 15 +-
.../persistence/graph/guice/GraphModule.java | 11 +-
.../graph/impl/GraphManagerImpl.java | 90 ++-
.../graph/impl/SimpleMarkedEdge.java | 62 +-
.../impl/stage/NodeDeleteListenerImpl.java | 27 +-
.../impl/EdgeSerializationImpl.java | 17 +-
.../impl/shard/AsyncTaskExecutor.java | 34 +
.../graph/serialization/impl/shard/Shard.java | 15 +
.../impl/shard/ShardEntryGroup.java | 13 +-
.../impl/shard/ShardGroupCompaction.java | 4 -
.../impl/shard/ShardGroupDeletion.java | 78 +++
.../impl/shard/impl/AsyncTaskExecutorImpl.java | 53 ++
.../shard/impl/NodeShardAllocationImpl.java | 81 ++-
.../shard/impl/ShardGroupColumnIterator.java | 72 +-
.../shard/impl/ShardGroupCompactionImpl.java | 10 +-
.../impl/shard/impl/ShardGroupDeletionImpl.java | 230 +++++++
.../impl/shard/impl/ShardsColumnIterator.java | 10 +
.../persistence/graph/GraphManagerIT.java | 76 +--
.../persistence/graph/GraphManagerLoadTest.java | 10 +-
.../graph/GraphManagerShardConsistencyIT.java | 382 ++++++++---
.../graph/GraphManagerStressTest.java | 16 +-
.../impl/shard/ShardEntryGroupTest.java | 14 +
.../impl/shard/ShardGroupCompactionTest.java | 30 +-
.../shard/impl/ShardGroupDeletionImplTest.java | 341 +++++++++
stack/corepersistence/pom.xml | 2 +-
.../persistence/index/CandidateResult.java | 11 +-
.../persistence/index/EntityIndexBatch.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 6 +
.../index/impl/DeIndexOperation.java | 5 +
.../index/impl/EsEntityIndexBatchImpl.java | 34 +-
.../index/impl/EsEntityIndexImpl.java | 188 +++--
.../persistence/index/impl/IndexingUtils.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 14 +-
.../usergrid/persistence/queue/QueueFig.java | 2 +-
.../org/apache/usergrid/rest/IndexResource.java | 350 ----------
.../apache/usergrid/rest/MigrateResource.java | 275 --------
.../apache/usergrid/rest/SystemResource.java | 111 ---
.../rest/applications/ApplicationResource.java | 50 ++
.../rest/applications/users/UserResource.java | 134 +++-
.../organizations/OrganizationsResource.java | 33 +
.../rest/system/ConnectionResource.java | 216 ++++++
.../usergrid/rest/system/DatabaseResource.java | 2 +
.../usergrid/rest/system/IndexResource.java | 353 ++++++++++
.../usergrid/rest/system/MigrateResource.java | 277 ++++++++
.../usergrid/rest/system/SystemResource.java | 121 ++++
.../collection/users/UserResourceIT.java | 117 ++++
stack/scripts/create_test_data.py | 222 ++++++
stack/scripts/migrate_entity_data.py | 18 +-
stack/scripts/multitenant_migrate.py | 683 +++++++++++++++++++
.../management/AppInfoMigrationPlugin.java | 5 +-
.../usergrid/management/ManagementService.java | 14 +-
.../cassandra/ManagementServiceImpl.java | 43 +-
81 files changed, 4231 insertions(+), 1445 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 14fbdaa,6b9abbc..e5c25fb
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@@ -315,25 -316,34 +315,31 @@@ public class AmazonAsyncEventService im
boolean validateEmptySets = true;
Observable<IndexOperationMessage> indexoperationObservable;
//merge each operation to a master observable;
- if (event instanceof EdgeDeleteEvent) {
- indexoperationObservable = handleEdgeDelete(message);
- } else if (event instanceof EdgeIndexEvent) {
- indexoperationObservable = handleEdgeIndex(message);
- } else if (event instanceof EntityDeleteEvent) {
- indexoperationObservable = handleEntityDelete(message);
- } else if (event instanceof EntityIndexEvent) {
- indexoperationObservable = handleEntityIndexUpdate(message);
- } else if (event instanceof InitializeApplicationIndexEvent) {
+ if ( event instanceof EdgeDeleteEvent ) {
+ indexoperationObservable = handleEdgeDelete( message );
+ }
+ else if ( event instanceof EdgeIndexEvent ) {
+ indexoperationObservable = handleEdgeIndex( message );
+ }
+ else if ( event instanceof EntityDeleteEvent ) {
+ indexoperationObservable = handleEntityDelete( message );
+ validateEmptySets = false; // do not check this one for an empty set b/c it can be empty
+
+ }
+ else if ( event instanceof EntityIndexEvent ) {
+ indexoperationObservable = handleEntityIndexUpdate( message );
+ }
+ else if ( event instanceof InitializeApplicationIndexEvent ) {
//does not return observable
- handleInitializeApplicationIndex( event, message );
- indexoperationObservable = Observable.just( new IndexOperationMessage() );
+ handleInitializeApplicationIndex(event, message);
+ indexoperationObservable = Observable.just(new IndexOperationMessage());
validateEmptySets = false; //do not check this one for an empty set b/c it will be empty.
- }
- else if ( event instanceof ElasticsearchIndexEvent ) {
- handleIndexOperation( ( ElasticsearchIndexEvent ) event );
- indexoperationObservable = Observable.just( new IndexOperationMessage() );
+ } else if (event instanceof ElasticsearchIndexEvent) {
+ handleIndexOperation((ElasticsearchIndexEvent) event);
+ indexoperationObservable = Observable.just(new IndexOperationMessage());
validateEmptySets = false; //do not check this one for an empty set b/c it will be empty.
- }
-
- else {
- throw new Exception( "Unknown EventType" );//TODO: print json instead
+ } else {
+ throw new Exception("Unknown EventType");//TODO: print json instead
}
//collect all of the
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 69d5e18,9d02717..c991b36
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@@ -70,10 -72,19 +72,19 @@@ public interface IndexProcessorFig exte
int getIndexQueueVisibilityTimeout();
/**
+ * The number of worker threads used when handing off messages from the SQS thread
+ */
+ @Default( "20" )
+ @Key( EVENT_CONCURRENCY_FACTOR )
+ int getEventConcurrencyFactor();
+
+
+
+ /**
* The number of worker threads used to read index write requests from the queue.
*/
- @Default( "8" )
- @Key( ELASTICSEARCH_WORKER_COUNT )
+ @Default("16")
+ @Key(ELASTICSEARCH_WORKER_COUNT)
int getWorkerCount();
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 7efe8e4,b2a1a2a..301a7ae
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@@ -218,10 -219,10 +219,10 @@@ public class IndexServiceImpl implement
//collect results into a single batch
.collect( () -> ei.createBatch(), ( batch, candidateResult ) -> {
logger.debug( "Deindexing on edge {} for entity {} added to batch",searchEdge , entityId );
- batch.deindex( searchEdge, candidateResult );
+ batch.deindex( candidateResult );
} )
//return the future from the batch execution
- .flatMap( batch ->Observable.just(batch.build()) );
+ .map( batch ->batch.build() );
return ObservableTimer.time(batches, indexTimer);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
index f8079e5,658d3eb..9ad90eb
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/ApplicationServiceIT.java
@@@ -85,15 -87,10 +86,16 @@@ public class ApplicationServiceIT exten
, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
Optional.<Edge>absent() );
- Iterator<Edge> results = graphManager.loadEdgesFromSource(simpleSearchByEdgeType).toBlocking().getIterator();
+ Iterator<MarkedEdge>
+ results = graphManager.loadEdgesFromSource(simpleSearchByEdgeType).toBlocking().getIterator();
if(results.hasNext()){
- Assert.fail("should be empty");
+ int i = 0;
+
+ while(results.hasNext()){
+ results.next();
+ i++;
+ }
+ Assert.fail("should be empty but has "+i);
}else{
Results searchCollection = entityManager.searchCollection(entityManager.getApplication(), "tests", Query.all());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/corepersistence/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 1ffcd02,a8fb751..a49e217
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -117,7 -136,13 +136,12 @@@ public class EsEntityIndexBatchImpl imp
return deindex( searchEdge, entity.getId(), entity.getVersion() );
}
+ @Override
+ public EntityIndexBatch deindex( final CandidateResult entity ) {
+
+ return deindexWithDocId(entity.getDocId());
+ }
-
@Override
public IndexOperationMessage build() {
return container;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
index e830876,162565f..769b836
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
@@@ -17,7 -17,9 +17,8 @@@
package org.apache.usergrid.rest.applications;
+ import com.fasterxml.jackson.databind.ObjectMapper;
-import com.sun.jersey.api.json.JSONWithPadding;
-import com.sun.jersey.api.view.Viewable;
+import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
import org.apache.amber.oauth2.common.error.OAuthError;
import org.apache.amber.oauth2.common.exception.OAuthProblemException;
import org.apache.amber.oauth2.common.message.OAuthResponse;
@@@ -47,9 -48,9 +48,10 @@@ import org.apache.usergrid.rest.excepti
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.oauth.ClientCredentialsInfo;
+import org.glassfish.jersey.server.mvc.Viewable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
@@@ -601,6 -601,54 +603,54 @@@ public class ApplicationResource extend
if(value==null){
throw new EntityNotFoundException("apigeeMobileConfig not found, it is possibly not enabled for your config.");
}
- return new JSONWithPadding( value, callback );
+ return value;
}
+
+ // Specifically require superuser access as this is setting app properties directly (only way to currently do this
+ // with Apigee's apigeeMobileConfig
+ @RequireOrganizationAccess
+ @POST
+ @Path("apm/apigeeMobileConfig")
+ @Consumes(APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
- public JSONWithPadding setAPMConfig( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback,
- Map<String, Object> json) throws Exception {
++ public String setAPMConfig( @Context UriInfo ui,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
++ Map<String, Object> json ) throws Exception {
+
+ if(json == null || json.size() < 1){
+ logger.error("Param {} cannot be null for POST apm/apigeeMobileConfig", APIGEE_MOBILE_APM_CONFIG_JSON_KEY);
+ throw new IllegalArgumentException("Request body cannot be empty and must include apigeeMobileConfig params");
+ }
+
+ final String requestAppUUID = (String) json.get("applicationUUID");
+ if(!requestAppUUID.equalsIgnoreCase(applicationId.toString())){
+ logger.error("Provided application UUID {} does not match actual application UUID {}",
+ requestAppUUID,
+ applicationId.toString());
+ throw new IllegalArgumentException(
+ String.format("Provided application UUID %s does not match actual application UUID %s",
+ requestAppUUID,
+ applicationId.toString())
+ );
+ }
+
+ final String apmConfig = new ObjectMapper().writeValueAsString(json);
+ if(logger.isDebugEnabled()){
+ logger.debug("Received request to set apigeeMobileConfig properties with: {}", apmConfig);
+ }
+
+
+ EntityManager em = emf.getEntityManager( applicationId );
+ em.setProperty(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
+ APIGEE_MOBILE_APM_CONFIG_JSON_KEY,
+ apmConfig
+ );
+
+ logger.info("Successfully set apigeeMobileConfig properties with: {}", apmConfig);
+
- return new JSONWithPadding(apmConfig, callback);
++ return apmConfig;
+
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index b84fc08,fb2962e..130658cd
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@@ -17,38 -17,63 +17,63 @@@
package org.apache.usergrid.rest.applications.users;
--import com.fasterxml.jackson.databind.JsonNode;
--import com.fasterxml.jackson.databind.ObjectMapper;
- import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
- import net.tanesha.recaptcha.ReCaptchaImpl;
- import net.tanesha.recaptcha.ReCaptchaResponse;
+ import java.util.Map;
+ import java.util.UUID;
+
+ import javax.ws.rs.Consumes;
+ import javax.ws.rs.DefaultValue;
+ import javax.ws.rs.FormParam;
+ import javax.ws.rs.GET;
+ import javax.ws.rs.POST;
+ import javax.ws.rs.PUT;
+ import javax.ws.rs.Path;
+ import javax.ws.rs.PathParam;
+ import javax.ws.rs.Produces;
+ import javax.ws.rs.QueryParam;
+ import javax.ws.rs.core.Context;
+ import javax.ws.rs.core.MediaType;
+ import javax.ws.rs.core.PathSegment;
+ import javax.ws.rs.core.Response;
+ import javax.ws.rs.core.UriInfo;
+
++import org.glassfish.jersey.server.mvc.Viewable;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.annotation.Scope;
+ import org.springframework.stereotype.Component;
+
import org.apache.amber.oauth2.common.exception.OAuthProblemException;
import org.apache.amber.oauth2.common.message.OAuthResponse;
import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.management.ActivationState;
+ import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.tokens.exceptions.TokenException;
- import org.glassfish.jersey.server.mvc.Viewable;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
- import org.springframework.context.annotation.Scope;
- import org.springframework.stereotype.Component;
- import javax.ws.rs.*;
- import javax.ws.rs.core.*;
- import java.util.Map;
- import java.util.UUID;
-import com.sun.jersey.api.json.JSONWithPadding;
-import com.sun.jersey.api.view.Viewable;
++import com.fasterxml.jackson.databind.JsonNode;
++import com.fasterxml.jackson.databind.ObjectMapper;
++import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
- import static javax.servlet.http.HttpServletResponse.*;
- import static org.apache.usergrid.security.shiro.utils.SubjectUtils.*;
+ import net.tanesha.recaptcha.ReCaptchaImpl;
+ import net.tanesha.recaptcha.ReCaptchaResponse;
+
+ import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
+ import static javax.servlet.http.HttpServletResponse.SC_FORBIDDEN;
+ import static javax.servlet.http.HttpServletResponse.SC_OK;
+
+ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.getSubjectUserId;
+ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.isApplicationAdmin;
+ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.isApplicationUser;
import static org.apache.usergrid.utils.ConversionUtils.string;
@@@ -143,9 -164,84 +168,84 @@@ public class UserResource extends Servi
management.setAppUserPassword( getApplicationId(), targetUserId, oldPassword, newPassword );
}
- return new JSONWithPadding( response, callback );
+ return response;
}
+ @GET
+ @RequireSystemAccess
+ @Path("credentials")
- public JSONWithPadding getUserCredentials(@QueryParam("callback") @DefaultValue("callback") String callback )
++ public ApiResponse getUserCredentials(@QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ logger.info( "UserResource.getUserCredentials" );
+
+
+ final ApiResponse response = createApiResponse();
+ response.setAction( "get user credentials" );
+
+ final UUID applicationId = getApplicationId();
+ final UUID targetUserId = getUserUuid();
+
+ if ( applicationId == null ) {
+ response.setError( "Application not found" );
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ if ( targetUserId == null ) {
+ response.setError( "User not found" );
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ final CredentialsInfo credentialsInfo = management.getAppUserCredentialsInfo( applicationId, targetUserId );
+
+
+ response.setProperty( "credentials", credentialsInfo );
+
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+
+ @PUT
+ @RequireSystemAccess
+ @Path("credentials")
- public JSONWithPadding setUserCredentials( @Context UriInfo ui, Map<String, Object> json,
++ public ApiResponse setUserCredentials( @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ logger.info( "UserResource.setUserCredentials" );
+
+ if ( json == null ) {
+ return null;
+ }
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "set user credentials" );
+ Map<String, Object> credentialsJson = ( Map<String, Object> ) json.get( "credentials" );
+
+
+ if ( credentialsJson == null ) {
+ throw new IllegalArgumentException( "credentials sub object is required" );
+ }
+
+ final CredentialsInfo credentials = CredentialsInfo.fromJson( credentialsJson );
+
+ UUID applicationId = getApplicationId();
+ UUID targetUserId = getUserUuid();
+
+ if ( targetUserId == null ) {
+ response.setError( "User not found" );
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ management.setAppUserCredentialsInfo( applicationId, targetUserId, credentials );
+
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
@POST
@Path("password")
@@@ -202,9 -292,7 +302,9 @@@
@POST
@Path("sendpin")
- public JSONWithPadding postSendPin( @Context UriInfo ui,
+ @JSONP
- @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ @Produces({ MediaType.APPLICATION_JSON, "application/javascript"})
+ public ApiResponse postSendPin( @Context UriInfo ui,
@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
return sendPin( ui, callback );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
index 28d8c87,ac07aaa..f48168e
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
@@@ -17,34 -17,29 +17,40 @@@
package org.apache.usergrid.rest.management.organizations;
-import java.util.*;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
+import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.management.ApplicationCreator;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.apache.usergrid.management.exceptions.ManagementException;
+import org.apache.usergrid.persistence.index.query.Identifier;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.RootResource;
import org.apache.usergrid.rest.management.ManagementResource;
+import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
++
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
-import org.apache.usergrid.management.ApplicationCreator;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.OrganizationOwnerInfo;
-import org.apache.usergrid.management.exceptions.ManagementException;
-import org.apache.usergrid.rest.AbstractContextResource;
-import org.apache.usergrid.rest.ApiResponse;
-import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
-import org.apache.commons.lang.StringUtils;
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
++
++import java.util.ArrayList;
++import java.util.HashMap;
++import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.usergrid.rest.exceptions.SecurityException.mappableSecurityException;
+import static org.apache.usergrid.security.shiro.utils.SubjectUtils.isPermittedAccessToOrganization;
@Component( "org.apache.usergrid.rest.management.organizations.OrganizationsResource" )
@@@ -67,7 -62,34 +73,34 @@@ public class OrganizationsResource exte
public OrganizationsResource() {
}
+
+ @GET
+ @RequireSystemAccess
- public JSONWithPadding getAllOrganizations() throws Exception{
++ public ApiResponse getAllOrganizations() throws Exception{
+
+ ApiResponse response = createApiResponse();
+ List<OrganizationInfo> orgs = management.getOrganizations(null, 10000);
+ List<Object> jsonOrgList = new ArrayList<>();
+
+ for(OrganizationInfo org: orgs){
+
+
+ Map<String, Object> jsonOrg = new HashMap<>();
+ Map<String, UUID> apps = management.getApplicationsForOrganization(org.getUuid()).inverse();
+
+ jsonOrg.put("name", org.getName());
+ jsonOrg.put("uuid", org.getUuid());
+ jsonOrg.put("properties", org.getProperties());
+ jsonOrg.put("applications", apps);
+ jsonOrgList.add(jsonOrg);
+ }
+
+ response.setProperty("organizations", jsonOrgList);
- return new JSONWithPadding(response);
++
++ return response;
+ }
+
@Path(RootResource.ORGANIZATION_ID_PATH)
- @RequireOrganizationAccess
public OrganizationResource getOrganizationById( @Context UriInfo ui,
@PathParam( "organizationId" ) String organizationIdStr )
throws Exception {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
index 0000000,14b79f3..ff2a739
mode 000000,100644..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
@@@ -1,0 -1,218 +1,216 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.usergrid.rest.system;
+
+
+ import java.util.HashMap;
+ import java.util.Map;
+ import java.util.UUID;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicLong;
+
+ import javax.ws.rs.DefaultValue;
+ import javax.ws.rs.GET;
+ import javax.ws.rs.POST;
+ import javax.ws.rs.Path;
+ import javax.ws.rs.PathParam;
+ import javax.ws.rs.Produces;
+ import javax.ws.rs.QueryParam;
+ import javax.ws.rs.core.MediaType;
+
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.annotation.Scope;
+ import org.springframework.stereotype.Component;
+
+ import org.apache.usergrid.corepersistence.service.ConnectionService;
+ import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
+ import org.apache.usergrid.corepersistence.service.StatusService;
+ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+ import org.apache.usergrid.persistence.index.query.Identifier;
+ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
+ import org.apache.usergrid.rest.RootResource;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+ import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
+
+ import rx.Observable;
-import rx.functions.Action1;
+ import rx.schedulers.Schedulers;
+
+
+ /**
+ * system/index/otherstuff
+ */
+ @Component
+ @Scope( "singleton" )
+ @Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+ } )
+ public class ConnectionResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( ConnectionResource.class );
+
+ public ConnectionResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "dedup/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ public ApiResponse rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding all applications" );
+
+ final UUID applicationId = UUIDUtils.tryGetUUID( applicationIdStr );
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+
+ return executeAndCreateResponse( applicationId, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "dedup/{jobId: " + Identifier.UUID_REX + "}" )
- public JSONWithPadding rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ public ApiResponse rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+ logger.info( "Getting status for index jobs" );
+
+ Preconditions.checkNotNull( jobId, "query param jobId must not be null" );
+
+
+ final UUID jobUUID = UUIDUtils.tryGetUUID( jobId );
+
+ final StatusService.JobStatus
+ job = getStatusService().getStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobUUID ).toBlocking().lastOrDefault(
+ null );
+
+ Preconditions.checkNotNull( job, "job with id '" + jobId + "' does not exist" );
+
+
+ return createResult( job, callback );
+ }
+
+
+ private ConnectionService getConnectionService() {
+ return injector.getInstance( ConnectionServiceImpl.class );
+ }
+
+
+ private StatusService getStatusService() {
+ return injector.getInstance( StatusService.class );
+ }
+
+
+
+ /**
+ * Execute the request and return the response.
+ */
- private JSONWithPadding executeAndCreateResponse( final UUID applicationId, final String callback ) {
++ private ApiResponse executeAndCreateResponse( final UUID applicationId, final String callback ) {
+
+ final Observable<ApplicationScope> applicationScopeObservable =
+ Observable.just( CpNamingUtils.getApplicationScope( applicationId ) );
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final StatusService statusService = getStatusService();
+ final ConnectionService connectionService = getConnectionService();
+
+ final AtomicLong count = new AtomicLong( 0 );
+
+ //start de duping and run in the background
+ connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
+ .doOnNext(buffer -> {
+
+
+ final long runningTotal = count.addAndGet(buffer.size());
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", System.currentTimeMillis());
+ }};
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
+ StatusService.Status.INPROGRESS, status).toBlocking().lastOrDefault(null);
+ }).doOnSubscribe(() -> {
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.STARTED, new HashMap<>()).toBlocking().lastOrDefault(null);
+
+ }).doOnCompleted(() -> {
+
+ final long runningTotal = count.get();
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put("countProcessed", runningTotal);
+ put("updatedTimestamp", System.currentTimeMillis());
+ }};
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.COMPLETE, status).toBlocking().lastOrDefault(null);
+
+ }).doOnError( (throwable) -> {
+ logger.error("Error deduping connections", throwable);
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put("error", throwable.getMessage() );
+ }};
+
+ statusService.setStatus(CpNamingUtils.MANAGEMENT_APPLICATION_ID,
+ jobId, StatusService.Status.FAILED, status).toBlocking().lastOrDefault(null);;
+
+ } ).subscribeOn(Schedulers.newThread()).subscribe();
+
+
+ final StatusService.JobStatus status =
+ new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
+
+ return createResult( status, callback );
+ }
+
+
+ /**
+ * Create a response with the specified data.
+ * @param jobStatus
+ * @param callback
+ * @return
+ */
- private JSONWithPadding createResult(final StatusService.JobStatus jobStatus, final String callback){
++ private ApiResponse createResult(final StatusService.JobStatus jobStatus, final String callback){
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "de-dup connections" );
+ response.setProperty( "status", jobStatus );
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+ }
+
+
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
index b261c96,42a63ca..f3ce8b7
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
@@@ -34,8 -31,12 +34,10 @@@ import org.slf4j.LoggerFactory
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-import com.sun.jersey.api.json.JSONWithPadding;
-
@Component
@Scope( "singleton" )
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
index 0000000,8e2946c..bbbe8b3
mode 000000,100644..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
@@@ -1,0 -1,331 +1,353 @@@
+ /*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+ package org.apache.usergrid.rest.system;
+
+
++import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+ import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
+ import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
+ import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl;
+ import org.apache.usergrid.corepersistence.index.ReIndexService;
+ import org.apache.usergrid.persistence.EntityManager;
+ import org.apache.usergrid.persistence.index.utils.ConversionUtils;
+ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
+ import org.apache.usergrid.rest.RootResource;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.annotation.Scope;
+ import org.springframework.stereotype.Component;
+
+ import javax.ws.rs.*;
+ import javax.ws.rs.core.Context;
+ import javax.ws.rs.core.MediaType;
+ import javax.ws.rs.core.UriInfo;
+ import java.util.HashMap;
+ import java.util.Map;
+ import java.util.UUID;
+
+
+ /**
+ * system/index/otherstuff
+ */
+ @Component
+ @Scope( "singleton" )
+ @Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+ } )
+ public class IndexResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( IndexResource.class );
+ private static final String UPDATED_FIELD = "updated";
+
+
+
+ public IndexResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild" )
- public JSONWithPadding rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ public ApiResponse rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info("Rebuilding all applications");
+
+ final ReIndexRequestBuilder request = createRequest();
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "rebuild/{jobId}" )
- public JSONWithPadding rebuildIndexesGet(@PathParam( "jobId" ) String jobId, @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ @JSONP
++ @Produces({ MediaType.APPLICATION_JSON, "application/javascript" })
++ public ApiResponse rebuildIndexesGet(
++ @PathParam( "jobId" ) String jobId,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++
++
+ throws Exception {
+ logger.info("Getting status for index jobs");
+
+ Preconditions
+ .checkNotNull(jobId, "query param jobId must not be null" );
+
+
+ ReIndexService.ReIndexStatus status = getReIndexService().getStatus(jobId);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild" )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Resuming rebuilding all applications" );
+ final ReIndexRequestBuilder request = createRequest();
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+
+ logger.info( "Rebuilding application {}", applicationIdStr );
+
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+ logger.info( "Resuming rebuilding application {}", applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ logger.info( "Resuming rebuilding collection {} in application {}", collectionName, applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/management" )
- public JSONWithPadding rebuildInternalIndexesPost(
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildInternalIndexesPost(
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception {
+
+
+ final UUID managementAppId = emf.getManagementAppId();
+
+ logger.info( "Rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/management" )
- public JSONWithPadding rebuildInternalIndexesPut( final Map<String, Object> payload,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse rebuildInternalIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" )
+ String callback ) throws Exception {
+
+
+ final UUID managementAppId = emf.getManagementAppId();
+
+ logger.info( "Resuming rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path(RootResource.APPLICATION_ID_PATH)
- public JSONWithPadding addIndex( @Context UriInfo ui,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse addIndex( @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ Preconditions
+ .checkNotNull( config, "Payload for config is null, please pass {replicas:int, shards:int} in body" );
+
+ ApiResponse response = createApiResponse();
+
+ if ( !config.containsKey( "replicas" ) || !config.containsKey( "shards" ) ||
+ !( config.get( "replicas" ) instanceof Integer ) || !( config.get( "shards" ) instanceof Integer ) ) {
+ throw new IllegalArgumentException( "body must contains 'replicas' of type int and 'shards' of type int" );
+ }
+
+ if ( !config.containsKey( "indexSuffix" ) ) {
+ throw new IllegalArgumentException( "Please add an indexSuffix to your post" );
+ }
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ if(appId == null){
+ throw new IllegalArgumentException("app id was not parsed");
+ }
+
+ EntityManager em = emf.getEntityManager(appId);
+ em.addIndex(config.get("indexSuffix").toString(), (int) config.get("shards"),
+ (int) config.get("replicas"), (String) config.get("writeConsistency"));
+ response.setAction( "Add index to alias" );
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ private ReIndexService getReIndexService() {
+ return injector.getInstance( ReIndexService.class );
+ }
+
+
+ private ReIndexRequestBuilder createRequest() {
+ //TODO: wire this up through spring, and in the future guice.
+ return new ReIndexRequestBuilderImpl();
+ }
+
+
- private JSONWithPadding executeResumeAndCreateResponse( final Map<String, Object> payload,
++ private ApiResponse executeResumeAndCreateResponse( final Map<String, Object> payload,
+ final ReIndexRequestBuilder request,
+ final String callback ) {
+
+ Map<String,Object> newPayload = payload;
+ if(newPayload == null || !payload.containsKey( UPDATED_FIELD )){
+ newPayload = new HashMap<>(1);
+ newPayload.put(UPDATED_FIELD,0);
+ }
+
+ Preconditions.checkArgument(newPayload.get(UPDATED_FIELD) instanceof Number,
+ "You must specified the field \"updated\" in the payload and it must be a timestamp" );
+
+ //add our updated timestamp to the request
+ if ( newPayload.containsKey( UPDATED_FIELD ) ) {
+ final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_FIELD));
+ request.withStartTimestamp( timestamp );
+ }
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ /**
+ * Execute the request and return the response.
+ */
- private JSONWithPadding executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
++ private ApiResponse executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
+
+
+ final ReIndexService.ReIndexStatus status = getReIndexService().rebuildIndex( request );
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
index 0000000,c5a6dbc..7a6100b
mode 000000,100644..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/MigrateResource.java
@@@ -1,0 -1,270 +1,277 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.usergrid.rest.system;
+
+
-import java.util.Map;
-import java.util.Set;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
++import com.fasterxml.jackson.databind.node.JsonNodeFactory;
++import com.fasterxml.jackson.databind.node.ObjectNode;
++import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
++import com.google.common.base.Preconditions;
++import com.google.inject.Injector;
+ import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+ import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
++import org.springframework.beans.factory.annotation.Autowired;
++import org.springframework.context.annotation.Scope;
++import org.springframework.stereotype.Component;
+
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import com.google.inject.Injector;
-import com.sun.jersey.api.json.JSONWithPadding;
++import javax.ws.rs.*;
++import javax.ws.rs.core.Context;
++import javax.ws.rs.core.MediaType;
++import javax.ws.rs.core.UriInfo;
++import java.util.Map;
++import java.util.Set;
+
+
+ @Component
+ @Scope( "singleton" )
+ @Produces( {
+ MediaType.APPLICATION_JSON,
+ "application/javascript",
+ "application/x-javascript",
+ "text/ecmascript",
+ "application/ecmascript",
+ "text/jscript"
+ } )
+ public class MigrateResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( MigrateResource.class );
+
+ public MigrateResource() {
+ logger.info( "SystemResource initialized" );
+ }
+
+ @Autowired
+ private Injector guiceInjector;
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "run" )
- public JSONWithPadding migrateData( @Context UriInfo ui,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse migrateData( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data" );
+ //TODO make this use the task scheduler
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+
+ logger.info( "Migrating Schema" );
+
+ try {
+ getMigrationManager().migrate();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data", e );
+ }
+
+ logger.info( "Migrating Data" );
+
+ try {
+ getDataMigrationManager().migrate();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data", e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats" );
+ migrate.setDaemon( true );
+ migrate.start();
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "run/{pluginName}" )
- public JSONWithPadding migrateData(@PathParam("pluginName") String pluginName , @Context UriInfo ui,
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse migrateData(@PathParam("pluginName") String pluginName , @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ if(!getDataMigrationManager().pluginExists(pluginName)){
+ throw new IllegalArgumentException("Plugin doesn't exits name:"+pluginName);
+ }
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Data: "+ pluginName );
+ //TODO make this use the task scheduler
+
+
+
+
+ final Thread migrate = new Thread() {
+
+ @Override
+ public void run() {
+
+ logger.info( "Migrating Data for plugin: " + pluginName );
+
+ try {
+ getDataMigrationManager().migrate(pluginName);
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to migrate data for plugin: " + pluginName, e );
+ }
+ }
+ };
+
+ migrate.setName( "Index migrate data formats: "+pluginName );
+ migrate.setDaemon( true );
+ migrate.start();
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "set" )
- public JSONWithPadding setMigrationVersion(
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse setMigrationVersion(
+ @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback )
+ throws Exception {
+
+ logger.debug( "setMigrationVersion" );
+
+ Preconditions.checkNotNull( json, "You must provide a json body" );
+ Preconditions.checkArgument( json.keySet().size() > 0, "You must specify at least one module and version" );
+
+ ApiResponse response = createApiResponse();
+ response.setAction("Set Migration Versions");
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ /**
+ * Set the migration version for the plugins specified
+ */
+ for ( final String key : json.keySet() ) {
+
+ int version = ( int ) json.get( key );
+
+ dataMigrationManager.resetToVersion(key, version);
+ }
+
+
+ /**
+ * Echo back a response of the current versions for all plugins
+ */
+ for(final String pluginName: plugins){
+ node.put(pluginName, dataMigrationManager.getCurrentVersion(pluginName));
+ }
+
+
+ response.setData( node );
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "status" )
- public JSONWithPadding migrateStatus(
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse migrateStatus(
+ @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Migrate Schema indexes" );
+
+ ObjectNode node = JsonNodeFactory.instance.objectNode();
+
+
+
+ final DataMigrationManager dataMigrationManager = getDataMigrationManager();
+
+ final Set<String> plugins = dataMigrationManager.getPluginNames();
+
+ for(final String pluginName: plugins){
+ node.put( pluginName, dataMigrationManager.getCurrentVersion( pluginName ) );
+ }
+
+ response.setData( node );
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "count" )
- public JSONWithPadding migrateCount(
++ @JSONP
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse migrateCount(
+ @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "Current entity count in system" );
+
+ response.setProperty( "count", emf.performEntityCount() );
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+ /**
+ * Get the schema migration manager
+ */
+ private MigrationManager getMigrationManager() {
+ return guiceInjector.getInstance( MigrationManager.class );
+ }
+
+ /**
+ * Get the Data migration manager
+ */
+ private DataMigrationManager getDataMigrationManager() {
+ return guiceInjector.getInstance( DataMigrationManager.class );
+ }
+ }
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
index 0000000,a5174e6..a83756d
mode 000000,100644..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/SystemResource.java
@@@ -1,0 -1,108 +1,121 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.usergrid.rest.system;
+
+
+ import javax.ws.rs.DefaultValue;
+ import javax.ws.rs.GET;
+ import javax.ws.rs.Path;
+ import javax.ws.rs.Produces;
+ import javax.ws.rs.QueryParam;
+ import javax.ws.rs.core.Context;
+ import javax.ws.rs.core.MediaType;
+ import javax.ws.rs.core.UriInfo;
+
++import org.apache.usergrid.rest.system.ApplicationsResource;
++import org.apache.usergrid.rest.system.DatabaseResource;
++import org.apache.usergrid.rest.system.QueueSystemResource;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ import org.springframework.context.annotation.Scope;
+ import org.springframework.stereotype.Component;
+
+ import org.apache.usergrid.rest.AbstractContextResource;
+ import org.apache.usergrid.rest.ApiResponse;
+ import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
-import com.sun.jersey.api.json.JSONWithPadding;
++import javax.ws.rs.*;
++import javax.ws.rs.core.Context;
++import javax.ws.rs.core.MediaType;
++import javax.ws.rs.core.UriInfo;
+
+
+ @Path( "/system" )
+ @Component
+ @Scope( "singleton" )
+ @Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+ } )
+ public class SystemResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( SystemResource.class );
+
+
+ public SystemResource() {
+ logger.info( "SystemResource initialized" );
+ }
+
+
+ @RequireSystemAccess
+ @GET
++
+ @Path( "superuser/setup" )
- public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
++ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
++ public ApiResponse getSetupSuperuser( @Context UriInfo ui,
++ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "superuser setup" );
+
+ logger.info( "Setting up Superuser" );
+
+ try {
+ management.provisionSuperuser();
+ }
+ catch ( Exception e ) {
+ logger.error( "Unable to complete superuser setup", e );
+ }
+
+ response.setSuccess();
+
- return new JSONWithPadding( response, callback );
++ return response;
+ }
+
+
+
+ @Path( "migrate" )
+ public MigrateResource migrate() {
+ return getSubResource( MigrateResource.class );
+ }
+
+
+ @Path( "index" )
+ public IndexResource index() { return getSubResource( IndexResource.class ); }
+
+
+ @Path( "database" )
+ public DatabaseResource database() {
+ return getSubResource( DatabaseResource.class );
+ }
+
++ @Path( "queue" )
++ public QueueSystemResource queue() {
++ return getSubResource( QueueSystemResource.class );
++ }
++
+ @Path( "applications" )
+ public ApplicationsResource applications() {
+ return getSubResource( ApplicationsResource.class );
+ }
+
+
+ @Path( "connection" )
+ public ConnectionResource connection() { return getSubResource( ConnectionResource.class ); }
+
+ }
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index 0df2416,f258f94..eddf7ed
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@@ -18,6 -18,6 +18,10 @@@ package org.apache.usergrid.rest.applic
import com.fasterxml.jackson.databind.JsonNode;
++import com.sun.jersey.api.client.UniformInterfaceException;
++import com.sun.jersey.api.client.WebResource;
++import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
++
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@@ -27,6 -27,6 +31,8 @@@ import java.util.UUID
import org.apache.usergrid.rest.test.resource.AbstractRestIT;
import org.apache.usergrid.rest.test.resource.endpoints.CollectionEndpoint;
import org.apache.usergrid.rest.test.resource.model.*;
++
++import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
@@@ -36,10 -36,16 +42,14 @@@ import org.slf4j.LoggerFactory
import org.apache.usergrid.rest.applications.utils.UserRepo;
import org.apache.usergrid.utils.UUIDUtils;
-import com.sun.jersey.api.client.ClientResponse.Status;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
-
++import javax.ws.rs.BadRequestException;
+import javax.ws.rs.ClientErrorException;
++import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
import java.io.IOException;
+ import javax.ws.rs.core.MediaType;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@@ -1129,4 -1109,105 +1139,111 @@@ public class UserResourceIT extends Abs
assertEquals(response.getResponse().getEntities().get(0).get("uuid").toString(), userId.toString());
}
+
+
+
+ @Test
+ public void testCredentialsTransfer() throws Exception {
+
+ usersResource.post(new User("test_1", "Test1 User", "test_1@test.com", "test123")); // client.setApiUrl(apiUrl);
+ refreshIndex();
+
+ //Entity appInfo = this.app().get().getResponse().getEntities().get(0);
+
+ Token token = this.app().token().post(new Token("test_1", "test123"));
+
+ assertNotNull(token.getAccessToken());
+
+ final String superUserName = this.clientSetup.getSuperuserName();
+ final String superUserPassword = this.clientSetup.getSuperuserPassword();
+
+
+ //get the credentials info
+ final CollectionEndpoint collection = userResource.entity("test_1").collection( "credentials" );
+
- final WebResource resource = collection.getResource();
++ final WebTarget resource = collection.getTarget();
++
++
++ final HttpAuthenticationFeature httpBasicAuth = HttpAuthenticationFeature.basicBuilder()
++ .credentials( superUserName, superUserPassword ).build();
+
- resource.addFilter( new HTTPBasicAuthFilter(superUserName, superUserPassword) );
+
++ final ApiResponse response = resource.register( httpBasicAuth ).request()
++ .accept( MediaType.APPLICATION_JSON ).get(
++ org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
+
- final ApiResponse response = resource.type( MediaType.APPLICATION_JSON_TYPE)
- .accept( MediaType.APPLICATION_JSON ).get( org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
+
+ //now get the credentials sub object
+
+ final Map<String, Object> credentials = ( Map<String, Object> ) response.getProperties().get( "credentials" );
+
+
+
+ //get out the hash and change it so we can validate
+ final String originalSecret = ( String ) credentials.get( "secret" );
+
+
+ //here we modify the hash a little, this way we can break password validation, then re-set it to ensure we're actually updating the credentials info correctly.
+ final String borkedSecret = originalSecret.substring( 0, originalSecret.length() -1 );
+
+ credentials.put( "secret", borkedSecret );
+
+ //now PUT it
+
+
+ final Map<String, Map<String, Object>> wrapper = new HashMap<>( );
+ wrapper.put( "credentials", credentials );
+
- final WebResource putResource = collection.getResource();
++ final WebTarget putResource = collection.getTarget();
+
- putResource.addFilter( new HTTPBasicAuthFilter(superUserName, superUserPassword) );
+
+
- putResource.type( MediaType.APPLICATION_JSON_TYPE)
- .accept( MediaType.APPLICATION_JSON ).put(
- org.apache.usergrid.rest.test.resource.model.ApiResponse.class, wrapper );
++ putResource.register( httpBasicAuth ).request()
++ .accept( MediaType.APPLICATION_JSON )
++ .put( javax.ws.rs.client.Entity.json(wrapper), org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
+
+ //now try to get a password, it should fail because the hash is no longer correct
+
+ int status = 0;
+
+ // bad access token
+ try {
+ this.app().token().post(new Token("test_1", "test123"));
+ fail("Should have thrown an exception");
- } catch (UniformInterfaceException uie) {
++ } catch (BadRequestException uie) {
+ status = uie.getResponse().getStatus();
- log.info("Error Response Body: " + uie.getResponse().getEntity(String.class));
++ log.info("Error Response Body: {}" , uie.getResponse().getEntity());
+ }
+
- assertEquals(Status.BAD_REQUEST.getStatusCode(), status);
++ assertEquals( Response.Status.BAD_REQUEST.getStatusCode(), status);
+
+
+ //now put the correct one
+
+
+ credentials.put( "secret", originalSecret );
+
+
- final WebResource putResource2 = collection.getResource();
++ final WebTarget putResource2 = collection.getTarget();
++
++
++
++ putResource2.register( httpBasicAuth ).request()
++ .accept( MediaType.APPLICATION_JSON )
++ .put( javax.ws.rs.client.Entity.json( wrapper ),
++ org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+
- putResource2.addFilter( new HTTPBasicAuthFilter( superUserName, superUserPassword ) );
+
+
- putResource2.type( MediaType.APPLICATION_JSON_TYPE)
- .accept( MediaType.APPLICATION_JSON ).put(
- org.apache.usergrid.rest.test.resource.model.ApiResponse.class, wrapper );
+
+
+ //now auth, should be good
+ final Token nextToken = this.app().token().post(new Token("test_1", "test123"));
+
+ assertNotNull( nextToken.getAccessToken() );
+
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b422364e/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
[04/50] [abbrv] usergrid git commit: Address issues found during code
review
Posted by to...@apache.org.
Address issues found during code review
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b1e1ed18
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b1e1ed18
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b1e1ed18
Branch: refs/heads/master
Commit: b1e1ed181d88f6a8e4b1b7bac76c99f936f66c8c
Parents: c9bf87c
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 22 12:13:00 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 22 12:13:00 2015 -0600
----------------------------------------------------------------------
.../graph/GraphManagerShardConsistencyIT.java | 5 +++--
.../impl/shard/impl/ShardGroupDeletionImplTest.java | 11 ++++++++---
2 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b1e1ed18/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index 9ee7b95..c1917bb 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -55,6 +55,7 @@ import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardCache;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -595,7 +596,7 @@ public class GraphManagerShardConsistencyIT {
while ( groups.hasNext() ) {
- group = groups.next();;
+ group = groups.next();
log.info( "Shard size for group is {}", group.getReadShards() );
@@ -604,7 +605,7 @@ public class GraphManagerShardConsistencyIT {
//we're done, 1 shard remains, we have a group, and it's our default shard
- if ( shardCount == 1 && group != null && group.getMinShard().getShardIndex() == 0 ) {
+ if ( shardCount == 1 && group != null && group.getMinShard().getShardIndex() == Shard.MIN_SHARD.getShardIndex() ) {
log.info( "All compactions complete," );
break;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b1e1ed18/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImplTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImplTest.java
index 9a3e407..f943e9b 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImplTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImplTest.java
@@ -43,6 +43,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
@@ -50,6 +51,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.same;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -252,7 +254,7 @@ public class ShardGroupDeletionImplTest {
//now check when marked we also retain them
final Iterator<MarkedEdge> markedEdgeIterator = Collections.singleton(
- ( MarkedEdge ) new SimpleMarkedEdge( createId( "source" ), "type", createId( "target" ), 1000, false ) )
+ ( MarkedEdge ) new SimpleMarkedEdge( createId( "source" ), "type", createId( "target" ), 1000, true ) )
.iterator();
@@ -266,7 +268,7 @@ public class ShardGroupDeletionImplTest {
@Test
- public void testDeletion() throws ExecutionException, InterruptedException {
+ public void testDeletion() throws ExecutionException, InterruptedException, ConnectionException {
final long createTime = 10000;
@@ -290,9 +292,10 @@ public class ShardGroupDeletionImplTest {
//mock up returning a mutation
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
+ final MutationBatch batch = mock( MutationBatch.class );
when( edgeShardSerialization.removeShardMeta( same( scope ), same( shard0 ), same( directedEdgeMeta ) ) )
- .thenReturn( mock( MutationBatch.class ) );
+ .thenReturn( batch );
final TimeService timeService = mock( TimeService.class );
@@ -310,6 +313,8 @@ public class ShardGroupDeletionImplTest {
final ShardGroupDeletion.DeleteResult result = future.get();
assertEquals( "should delete", ShardGroupDeletion.DeleteResult.DELETED, result );
+
+ verify(batch).execute();
}
[19/50] [abbrv] usergrid git commit: Add endpoint which lists all
orgs in the system (up to 10k) and provides all apps within the org as well.
Posted by to...@apache.org.
Add endpoint which lists all orgs in the system (up to 10k) and provides all apps within the org as well.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/752bf4e3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/752bf4e3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/752bf4e3
Branch: refs/heads/master
Commit: 752bf4e3fcb5e9b6064cac477bbde68a6c8727f3
Parents: 5eed978
Author: Michael Russo <mi...@gmail.com>
Authored: Mon Oct 26 15:24:11 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Mon Oct 26 15:24:11 2015 -0700
----------------------------------------------------------------------
.../organizations/OrganizationsResource.java | 42 +++++++++++++-------
1 file changed, 28 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/752bf4e3/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
index 4eac80d..ac07aaa 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
@@ -17,23 +17,14 @@
package org.apache.usergrid.rest.management.organizations;
-import java.util.Map;
-import java.util.UUID;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.FormParam;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import java.util.*;
+import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
-
import org.apache.usergrid.rest.RootResource;
import org.apache.usergrid.rest.management.ManagementResource;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -46,9 +37,7 @@ import org.apache.usergrid.management.exceptions.ManagementException;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
-
import org.apache.commons.lang.StringUtils;
-
import com.google.common.base.Preconditions;
import com.sun.jersey.api.json.JSONWithPadding;
@@ -74,6 +63,31 @@ public class OrganizationsResource extends AbstractContextResource {
}
+ @GET
+ @RequireSystemAccess
+ public JSONWithPadding getAllOrganizations() throws Exception{
+
+ ApiResponse response = createApiResponse();
+ List<OrganizationInfo> orgs = management.getOrganizations(null, 10000);
+ List<Object> jsonOrgList = new ArrayList<>();
+
+ for(OrganizationInfo org: orgs){
+
+
+ Map<String, Object> jsonOrg = new HashMap<>();
+ Map<String, UUID> apps = management.getApplicationsForOrganization(org.getUuid()).inverse();
+
+ jsonOrg.put("name", org.getName());
+ jsonOrg.put("uuid", org.getUuid());
+ jsonOrg.put("properties", org.getProperties());
+ jsonOrg.put("applications", apps);
+ jsonOrgList.add(jsonOrg);
+ }
+
+ response.setProperty("organizations", jsonOrgList);
+ return new JSONWithPadding(response);
+ }
+
@Path(RootResource.ORGANIZATION_ID_PATH)
@RequireOrganizationAccess
public OrganizationResource getOrganizationById( @Context UriInfo ui,
[11/50] [abbrv] usergrid git commit: De-couple entity deletes from
edge removal from graph.
Posted by to...@apache.org.
De-couple entity deletes from edge removal from graph.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/032ffd48
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/032ffd48
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/032ffd48
Branch: refs/heads/master
Commit: 032ffd48127e600d026446535cfb776f92aac000
Parents: d35fea5
Author: Michael Russo <mi...@gmail.com>
Authored: Fri Oct 23 12:59:57 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Fri Oct 23 12:59:57 2015 -0700
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 7 +++---
.../asyncevents/EventBuilder.java | 25 +++++++++++++++-----
.../asyncevents/EventBuilderImpl.java | 10 ++++----
.../asyncevents/InMemoryAsyncEventService.java | 5 ++--
4 files changed, 31 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/032ffd48/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 7034a67..fe53776 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -586,9 +586,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
- entityDeleteResults
- .getEntitiesCompacted()
- .collect(() -> new ArrayList<>(), (list, item) -> list.add(item)).toBlocking().lastOrDefault(null);
+ // Delete the entities and remove from graph separately
+ entityDeleteResults.getEntitiesDeleted().toBlocking().lastOrDefault(null);
+
+ entityDeleteResults.getCompactedNode().toBlocking().lastOrDefault(null);
return entityDeleteResults.getIndexObservable();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/032ffd48/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
index d246e2f..480756f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
@@ -80,17 +80,25 @@ public interface EventBuilder {
/**
* A bean to hold both our observables so the caller can choose the subscription mechanism. Note that
- * indexOperationMessages should be subscribed and completed BEFORE the getEntitiesCompacted is subscribed
+ * indexOperationMessages should be subscribed and completed BEFORE the getEntitiesDeleted is subscribed
*/
final class EntityDeleteResults {
private final Observable<IndexOperationMessage> indexOperationMessageObservable;
- private final Observable<List<MvccLogEntry>> entitiesCompacted;
+ private final Observable<List<MvccLogEntry>> entitiesDeleted;
+
+
+
+ private final Observable<Id> compactedNode;
+
+
public EntityDeleteResults( final Observable<IndexOperationMessage> indexOperationMessageObservable,
- final Observable<List<MvccLogEntry>> entitiesCompacted ) {
+ final Observable<List<MvccLogEntry>> entitiesDeleted,
+ final Observable<Id> compactedNode) {
this.indexOperationMessageObservable = indexOperationMessageObservable;
- this.entitiesCompacted = entitiesCompacted;
+ this.entitiesDeleted = entitiesDeleted;
+ this.compactedNode = compactedNode;
}
@@ -98,9 +106,14 @@ public interface EventBuilder {
return indexOperationMessageObservable;
}
+ public Observable<List<MvccLogEntry>> getEntitiesDeleted() {
+ return entitiesDeleted;
+ }
- public Observable<List<MvccLogEntry>> getEntitiesCompacted() {
- return entitiesCompacted;
+ public Observable<Id> getCompactedNode() {
+ return compactedNode;
}
+
+
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/032ffd48/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index cc0356b..d819f39 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -128,8 +128,6 @@ public class EventBuilderImpl implements EventBuilder {
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- //needs get versions here.
-
//TODO: change this to be an observable
//so we get these versions and loop through them until we find the MvccLogEntry that is marked as delete.
@@ -155,10 +153,14 @@ public class EventBuilderImpl implements EventBuilder {
//observable of entries as the batches are deleted
final Observable<List<MvccLogEntry>> entries =
ecm.getVersions( entityId ).buffer( serializationFig.getBufferSize() )
- .doOnNext( buffer -> ecm.delete( buffer ) ).doOnCompleted( () -> gm.compactNode( entityId ) );
+ .doOnNext( buffer -> ecm.delete( buffer ) );
+
+
+ // observable of the edge delete from graph
+ final Observable<Id> compactedNode = gm.compactNode(entityId);
- return new EntityDeleteResults( edgeObservable, entries );
+ return new EntityDeleteResults( edgeObservable, entries, compactedNode );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/032ffd48/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
index d5a0398..fc6385c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/InMemoryAsyncEventService.java
@@ -22,8 +22,6 @@ package org.apache.usergrid.corepersistence.asyncevents;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.index.impl.IndexProducer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
@@ -103,7 +101,8 @@ public class InMemoryAsyncEventService implements AsyncEventService {
eventBuilder.buildEntityDelete( applicationScope, entityId );
run( results.getIndexObservable() );
- run( results.getEntitiesCompacted() );
+ run( results.getEntitiesDeleted() );
+ run( results.getCompactedNode() );
}
[07/50] [abbrv] usergrid git commit: Added trace statements to the
shard group deletion
Posted by to...@apache.org.
Added trace statements to the shard group deletion
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/db852d0a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/db852d0a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/db852d0a
Branch: refs/heads/master
Commit: db852d0a1d00f8ea89eddb82d4075b1b774d9673
Parents: 0604247
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 22 14:14:43 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 22 14:14:43 2015 -0600
----------------------------------------------------------------------
.../impl/shard/impl/ShardGroupDeletionImpl.java | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/db852d0a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index 38a7834..6d2a009 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -125,10 +125,13 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
final ShardEntryGroup shardEntryGroup,
final Iterator<MarkedEdge> edgeIterator ) {
+ logger.trace( "Beginning audit of shard group {}", shardEntryGroup );
+
/**
* Compaction is pending, we cannot check it
*/
if ( shardEntryGroup.isCompactionPending() ) {
+ logger.trace( "Shard group {} is compacting, not auditing group", shardEntryGroup );
return DeleteResult.COMPACTION_PENDING;
}
@@ -136,6 +139,7 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
final long currentTime = timeService.getCurrentTime();
if ( shardEntryGroup.isNew( currentTime ) ) {
+ logger.trace( "Shard group {} contains a shard that is is too new, not auditing group", shardEntryGroup );
return DeleteResult.TOO_NEW;
}
@@ -143,6 +147,8 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
* We have edges, and therefore cannot delete them
*/
if ( edgeIterator.hasNext() ) {
+ logger.trace( "Shard group {} has edges, not deleting", shardEntryGroup );
+
return DeleteResult.CONTAINS_EDGES;
}
@@ -157,6 +163,7 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
//skip the min shard
if(shard.isMinShard()){
+ logger.trace( "Shard {} in group {} is the minimum, not deleting", shard, shardEntryGroup );
continue;
}
@@ -173,6 +180,8 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
}
result = DeleteResult.DELETED;
+
+ logger.trace( "Removing shard {} in group {}", shard, shardEntryGroup );
}
@@ -187,6 +196,8 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
}
}
+ logger.trace( "Completed auditing shard group {}", shardEntryGroup );
+
return result;
}
[41/50] [abbrv] usergrid git commit: Rename new script to
multitenant-migrate and restore old script.
Posted by to...@apache.org.
Rename new script to multitenant-migrate and restore old script.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8af152e6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8af152e6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8af152e6
Branch: refs/heads/master
Commit: 8af152e6bef74aa6259aff5a69428523da35d0bf
Parents: 57a613b
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 18:00:54 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 18:00:54 2015 -0400
----------------------------------------------------------------------
stack/scripts/migrate_entity_data.py | 401 +++++------------
stack/scripts/multitenant_migrate.py | 703 ++++++++++++++++++++++++++++++
2 files changed, 816 insertions(+), 288 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8af152e6/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index 0edd319..9c01270 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -16,64 +16,30 @@
# under the License.
#
#
-# To migrate multiple tenants within one cluster.
#
-# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
+# Usage from a machine running Usergrid with the new Usergrid version:
#
-# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+# ######################################################
+# STEP 1 - BEFORE SWITCHING TRAFFIC TO NEW UG VERSION
+# ######################################################
#
-# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --init
+# python migrate_entity_data.py --user adminuser:adminpass
#
-# This command will setup and bootstrap the database, setup the migration system and update index mappings:
-# - /system/database/setup
-# - /system/database/bootstrap
-# - /system/migrate/run/migration-system
-# - /system/migrate/run/index_mapping_migration
+# The above command performs an appinfo migration and system re-index only. This creates indices in Elasticsearch with
+# the updated indexing strategy in the new Usergrid version.
#
-# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
-# it will de-dup connections and re-index the app.
+# ######################################################
+# STEP 2 - AFTER SWITCHING TRAFFIC TO NEW UG VERSION
+# ######################################################
#
-# Write down the 'Re-index start' timestamp when this is finished.
+# python migrate_entity_data.py --user adminuser:adminpass --delta --date <timestamp>
#
-# STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
-#
-# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
-#
-# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
-#
-# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
-# it will de-dup connections and re-index the app with a start-date specified so only data modified since
-# STEP 1 will be re-indexed.
-#
-# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
-#
-# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
-#
-# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass>
-#
-# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
-# it will de-dup connections and re-index the app.
-#
-# Write down the 'Re-index start' timestamp when this is finished.
+# The above command performs an appinfo migration, system re-index using a start date, and full data migration which
+# includes entity data. This step is necessary to ensure Usergrid starts reading and writing data from the latest
+# entity version, including delta indexing of any documents create during the time between STEP 1 and STEP 2. If
+# all data has already been migrated (running this a 2nd, 3rd, etc. time), then the appinfo migration will be skipped.
+
-# STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
-#
-# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
-#
-# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
-#
-# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
-# it will de-dup connections and re-index the app with a start-date specified so only data modified since
-# STEP 1 will be re-indexed.
-#
-# STEP 5 - FULL DATA MIGRATION
-#
-# Login to any Tomcat instance in the cluster and run this command (admin user creds must be specificed but will be ignored):
-#
-# python migrate_entity_data.py --super <user>:<pass> --admin <user>:<pass> --full
-#
-# This command will run the full data migration.
-#
import sys
import logging
@@ -101,66 +67,40 @@ PLUGIN_ENTITYDATA = 'collections-entity-data'
PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
PLUGIN_CORE_DATA = 'core-data'
-MANAGEMENT_APP_ID = 'b6768a08-b5d5-11e3-a495-11ddb1de66c8'
-
def parse_args():
parser = argparse.ArgumentParser(description='Usergrid Migration Tool')
+ parser.add_argument('--date',
+ help='A date from which to start the migration',
+ type=str)
+
parser.add_argument('--endpoint',
help='The endpoint to use for making API requests.',
type=str,
default='http://localhost:8080')
- parser.add_argument('--super',
- help='Superuser username and creds <user:pass>',
+ parser.add_argument('--user',
+ help='System Admin Credentials used to authenticate with Usergrid <user:pass>',
type=str,
required=True)
- parser.add_argument('--admin',
- help='Organization admin creds <user:pass>',
- type=str,
- required=True)
-
- parser.add_argument('--init',
- help='Init system and start first migration.',
- action='store_true',
- default=False)
-
- parser.add_argument('--org',
- help='Name of organization on which to run migration.',
- type=str,
- required=False)
-
- parser.add_argument('--date',
- help='A date from which to start the migration',
- type=str)
-
- parser.add_argument('--full',
- help='Run full data migration (last step in cluster migration).',
+ parser.add_argument('--delta',
+ help='Run a delta migration.',
action='store_true',
default=False)
my_args = parser.parse_args(sys.argv[1:])
arg_vars = vars(my_args)
-
- creds = arg_vars['super'].split(':')
- if len(creds) != 2:
- print('Superuser credentials not properly specified. Must be "-u <user:pass>". Exiting...')
- exit_on_error()
- else:
- arg_vars['superuser'] = creds[0]
- arg_vars['superpass'] = creds[1]
-
- creds = arg_vars['super'].split(':')
+ creds = arg_vars['user'].split(':')
if len(creds) != 2:
- print('Org admin credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ print('Credentials not properly specified. Must be "-u <user:pass>". Exiting...')
exit_on_error()
else:
- arg_vars['adminuser'] = creds[0]
- arg_vars['adminpass'] = creds[1]
+ arg_vars['user'] = creds[0]
+ arg_vars['pass'] = creds[1]
return arg_vars
@@ -178,13 +118,9 @@ class Migrate:
'full_data_migration_start': '',
'full_data_migration_end': ''}
self.logger = init_logging(self.__class__.__name__)
- self.super_user = self.args['superuser']
- self.super_pass = self.args['superpass']
- self.admin_user = self.args['adminuser']
- self.admin_pass = self.args['adminpass']
- self.org = self.args['org']
- self.init = self.args['init']
- self.full = self.args['full']
+ self.admin_user = self.args['user']
+ self.admin_pass = self.args['pass']
+ self.delta_migration = self.args['delta']
def run(self):
self.logger.info('Initializing...')
@@ -197,83 +133,63 @@ class Migrate:
try:
- if self.full:
+ self.run_database_setup()
- # Do full data migration and exit
+ # We need to check and roll the migration system to 1 if not already
+ migration_system_updated = self.is_migration_system_updated()
- self.start_fulldata_migration()
-
- self.metrics['full_data_migration_start'] = get_current_time()
- self.logger.info("Full Data Migration Started")
- is_migrated = False
- while not is_migrated:
+ if not migration_system_updated:
+ self.logger.info('Migration system needs to be updated. Updating migration system..')
+ self.start_migration_system_update()
+ while not migration_system_updated:
time.sleep(STATUS_INTERVAL_SECONDS)
- is_migrated = self.is_data_migrated()
- if is_migrated:
+ migration_system_updated = self.is_migration_system_updated()
+ if migration_system_updated:
break
- self.metrics['full_data_migration_end'] = get_current_time()
- self.logger.info("Full Data Migration completed")
+ index_mapping_updated = self.is_index_mapping_updated()
- self.log_metrics()
- self.logger.info("Finished...")
-
- return
-
-
- if self.init:
-
- # Init the migration system as this is the first migration done on the cluster
-
- self.run_database_setup()
-
- migration_system_updated = self.is_migration_system_updated()
-
- if not migration_system_updated:
- self.logger.info('Migration system needs to be updated. Updating migration system..')
- self.start_migration_system_update()
- while not migration_system_updated:
- time.sleep(STATUS_INTERVAL_SECONDS)
- migration_system_updated = self.is_migration_system_updated()
- if migration_system_updated:
- break
+ if not index_mapping_updated:
+ self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
+ self.start_index_mapping_migration()
+ while not index_mapping_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ index_mapping_updated = self.is_index_mapping_updated()
+ if index_mapping_updated:
+ break
- index_mapping_updated = self.is_index_mapping_updated()
+ # Run AppInfo migration only when both appinfos and collection entity data have not been migrated
+ if not self.is_data_migrated():
- if not index_mapping_updated:
- self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
- self.start_index_mapping_migration()
- while not index_mapping_updated:
- time.sleep(STATUS_INTERVAL_SECONDS)
- index_mapping_updated = self.is_index_mapping_updated()
- if index_mapping_updated:
- break
+ #Migrate app info
+ if self.is_appinfo_migrated():
+ self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
+ self.reset_appinfo_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ self.start_appinfo_migration()
+ self.logger.info('AppInfo Migration Started.')
+ self.metrics['appinfo_migration_start'] = get_current_time()
- # Migrate app info
+ is_appinfo_migrated = False
+ while not is_appinfo_migrated:
+ is_appinfo_migrated = self.is_appinfo_migrated()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ if is_appinfo_migrated:
+ self.metrics['appinfo_migration_end'] = get_current_time()
+ break
+ self.logger.info('AppInfo Migration Ended.')
- if self.is_appinfo_migrated():
- self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
- self.reset_appinfo_migration()
- time.sleep(STATUS_INTERVAL_SECONDS)
- self.start_appinfo_migration()
- self.logger.info('AppInfo Migration Started.')
- self.metrics['appinfo_migration_start'] = get_current_time()
+ else:
+ self.logger.info('Full Data Migration previously ran... skipping AppInfo migration.')
- is_appinfo_migrated = False
- while not is_appinfo_migrated:
- is_appinfo_migrated = self.is_appinfo_migrated()
- time.sleep(STATUS_INTERVAL_SECONDS)
- if is_appinfo_migrated:
- self.metrics['appinfo_migration_end'] = get_current_time()
- break
- self.logger.info('AppInfo Migration Ended.')
- # Reindex management app
+ # We need to check and roll index mapping version to 1 if not already there
- job = self.start_app_reindex(MANAGEMENT_APP_ID)
+ # Perform system re-index (it will grab date from input if provided)
+ job = self.start_reindex()
self.metrics['reindex_start'] = get_current_time()
self.logger.info('Started Re-index. Job=[%s]', job)
is_running = True
@@ -286,44 +202,33 @@ class Migrate:
self.logger.info("Finished Re-index. Job=[%s]", job)
self.metrics['reindex_end'] = get_current_time()
+ # Only when we do a delta migration do we run the full data migration (includes appinfo and entity data)
+ if self.delta_migration:
- # Dedup and re-index all of organization's apps
-
- app_ids = self.get_app_ids()
- for app_id in app_ids:
-
- # De-dep app
- # job = self.start_dedup(app_id)
- # self.metrics['dedup_start_' + app_id] = get_current_time()
- # self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
- # is_running = True
- # while is_running:
- # time.sleep(STATUS_INTERVAL_SECONDS)
- # is_running = self.is_dedup_running(job)
- # if not is_running:
- # break
- #
- # self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
- # self.metrics['dedup_end_' + app_id] = get_current_time()
-
- # Re-index app
- job = self.start_app_reindex(app_id)
- self.metrics['reindex_start_' + app_id] = get_current_time()
- self.logger.info('Started Re-index. App=[%s], Job=[%s]', app_id, job)
- is_running = True
- while is_running:
+ self.logger.info('Delta option provided. Performing full data migration...')
+ if self.is_data_migrated():
+ self.reset_data_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ self.is_data_migrated()
+
+ # self.start_core_data_migration()
+ self.start_fulldata_migration()
+
+ self.metrics['full_data_migration_start'] = get_current_time()
+ self.logger.info("Full Data Migration Started")
+ is_migrated = False
+ while not is_migrated:
time.sleep(STATUS_INTERVAL_SECONDS)
- is_running = self.is_reindex_running(job)
- if not is_running:
+ is_migrated = self.is_data_migrated()
+ if is_migrated:
break
- self.logger.info("Finished Re-index. App=[%s], Job=[%s]", app_id, job)
- self.metrics['reindex_end_' + app_id] = get_current_time()
+ self.metrics['full_data_migration_end'] = get_current_time()
+ self.logger.info("Full Data Migration completed")
self.log_metrics()
self.logger.info("Finished...")
-
except KeyboardInterrupt:
self.log_metrics()
self.logger.error('Keyboard interrupted migration. Please run again to ensure the migration finished.')
@@ -332,10 +237,6 @@ class Migrate:
url = self.endpoint + '/system/database/setup'
return url
- def get_database_bootstrap_url(self):
- url = self.endpoint + '/system/database/bootstrap'
- return url
-
def get_migration_url(self):
url = self.endpoint + '/system/migrate/run'
return url
@@ -348,30 +249,28 @@ class Migrate:
url = self.endpoint + '/system/migrate/status'
return url
- def get_dedup_url(self):
- url = self.endpoint + '/system/connection/dedup'
- return url
-
def get_reindex_url(self):
url = self.endpoint + '/system/index/rebuild'
return url
def get_management_reindex_url(self):
- url = self.get_reindex_url() + "/management"
- return url
+ url = self.get_reindex_url() + "/management"
+ return url
+
def start_core_data_migration(self):
- try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
- response = r.json()
- return response
- except requests.exceptions.RequestException as e:
- self.logger.error('Failed to start migration, %s', e)
- exit_on_error(str(e))
+ try:
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
def start_fulldata_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -380,9 +279,9 @@ class Migrate:
def start_migration_system_update(self):
try:
- # TODO fix this URL
+ #TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -392,7 +291,7 @@ class Migrate:
def run_database_setup(self):
try:
setupUrl = self.get_database_setup_url()
- r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
+ r = requests.put(url=setupUrl, auth=(self.admin_user, self.admin_pass))
if r.status_code != 200:
exit_on_error('Database Setup Failed')
@@ -400,21 +299,10 @@ class Migrate:
self.logger.error('Failed to run database setup, %s', e)
exit_on_error(str(e))
- def run_database_bootstrap(self):
- try:
- setupUrl = self.get_database_bootstrap_url()
- r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
- if r.status_code != 200:
- exit_on_error('Database Bootstrap Failed')
-
- except requests.exceptions.RequestException as e:
- self.logger.error('Failed to run database bootstrap, %s', e)
- exit_on_error(str(e))
-
def start_index_mapping_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
- r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -435,7 +323,7 @@ class Migrate:
version = TARGET_ENTITY_DATA_VERSION - 1
body = json.dumps({PLUGIN_ENTITYDATA: version, PLUGIN_APPINFO: version})
try:
- r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
response = r.json()
self.logger.info('Resetting data migration versions to %s=[%s] '
'and %s=[%s]', PLUGIN_ENTITYDATA, version, PLUGIN_APPINFO, version)
@@ -448,7 +336,7 @@ class Migrate:
version = TARGET_APPINFO_VERSION - 1
body = json.dumps({PLUGIN_APPINFO: version})
try:
- r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
response = r.json()
self.logger.info('Resetting appinfo migration versions to %s=[%s]', PLUGIN_APPINFO, version)
return response
@@ -527,7 +415,7 @@ class Migrate:
def check_data_migration_status(self):
try:
- r = requests.get(url=self.get_migration_status_url(), auth=(self.super_user, self.super_pass))
+ r = requests.get(url=self.get_migration_status_url(), auth=(self.admin_user, self.admin_pass))
if r.status_code == 200:
response = r.json()
return response
@@ -542,20 +430,20 @@ class Migrate:
status_url = self.get_reindex_url()+'/' + job
try:
- r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
+ r = requests.get(url=status_url, auth=(self.admin_user, self.admin_pass))
response = r.json()
return response['status']
except requests.exceptions.RequestException as e:
self.logger.error('Failed to get reindex status, %s', e)
# exit_on_error()
- def start_app_reindex(self, appId):
+ def start_reindex(self):
body = ""
if self.start_date is not None:
body = json.dumps({'updated': self.start_date})
try:
- r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.super_user, self.super_pass))
+ r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.admin_user, self.admin_pass))
if r.status_code == 200:
response = r.json()
@@ -575,41 +463,6 @@ class Migrate:
else:
return False
- def get_dedup_status(self, job):
- status_url = self.get_dedup_url()+'/' + job
- try:
- r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
- print r.text
- response = r.json()
- return response['status']
- except requests.exceptions.RequestException as e:
- self.logger.error('Failed to get dedup status, %s', e)
- # exit_on_error()
-
- def start_dedup(self, app_id):
- body = ""
- try:
- r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.super_user, self.super_pass))
- if r.status_code == 200:
- response = r.json()
- print r.text
- return response['status']['jobStatusId']
- else:
- self.logger.error('Failed to start dedup, %s', r)
- exit_on_error(str(r))
-
- except requests.exceptions.RequestException as e:
- self.logger.error('Unable to make API request for dedup, %s', e)
- exit_on_error(str(e))
-
- def is_dedup_running(self, job):
- status = self.get_dedup_status(job)
- self.logger.info('Dedup status=[%s]', status)
- if status != "COMPLETE":
- return True
- else:
- return False
-
def is_endpoint_available(self):
try:
@@ -637,34 +490,6 @@ class Migrate:
)
- def get_app_ids(self):
-
- try:
-
- url = self.endpoint + "/management/token"
- body = json.dumps({"grant_type":"password","username":self.admin_user,"password":self.admin_pass})
- r = requests.post(url=url, data=body)
- if ( r.status_code != 200 ):
- print "Error logging in: " + r.text
- return
-
- access_token = r.json()["access_token"]
-
- url = self.endpoint + "/management/orgs/" + self.org + "/apps?access_token=" + access_token
- r = requests.get(url=url)
- if r.status_code != 200:
- exit_on_error('Cannot get app ids: ' + r.text)
-
- apps = r.json()["data"]
- app_ids = []
- for appId in apps.values():
- app_ids.append(appId)
-
- return app_ids;
-
- except requests.exceptions.RequestException as e:
- self.logger.error('Unable to get list of application ids, %s', e)
- exit_on_error(str(e))
def get_current_time():
return str(int(time.time()*1000))
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8af152e6/stack/scripts/multitenant_migrate.py
----------------------------------------------------------------------
diff --git a/stack/scripts/multitenant_migrate.py b/stack/scripts/multitenant_migrate.py
new file mode 100644
index 0000000..62e46af
--- /dev/null
+++ b/stack/scripts/multitenant_migrate.py
@@ -0,0 +1,703 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+# To migrate multiple tenants within one cluster.
+#
+# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
+#
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+#
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --init
+#
+# This command will setup and bootstrap the database, setup the migration system and update index mappings:
+# - /system/database/setup
+# - /system/database/bootstrap
+# - /system/migrate/run/migration-system
+# - /system/migrate/run/index_mapping_migration
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app.
+#
+# Write down the 'Re-index start' timestamp when this is finished.
+#
+# STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
+#
+# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
+#
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app with a start-date specified so only data modified since
+# STEP 1 will be re-indexed.
+#
+# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+#
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+#
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass>
+#
+# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app.
+#
+# Write down the 'Re-index start' timestamp when this is finished.
+
+# STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
+#
+# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
+#
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
+#
+# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
+# it will de-dup connections and re-index the app with a start-date specified so only data modified since
+# STEP 1 will be re-indexed.
+#
+# STEP 5 - FULL DATA MIGRATION
+#
+# Login to any Tomcat instance in the cluster and run this command (admin user creds must be specificed but will be ignored):
+#
+# python migrate_entity_data.py --super <user>:<pass> --admin <user>:<pass> --full
+#
+# This command will run the full data migration.
+#
+
+import sys
+import logging
+from logging.handlers import RotatingFileHandler
+import argparse
+import time
+import requests
+import json
+
+
+# Version expected in status response post-migration for entity and app-info data
+TARGET_APPINFO_VERSION=2
+TARGET_ENTITY_DATA_VERSION=2
+TARGET_CORE_DATA_VERSION=2
+TARGET_MIGRATION_SYSTEM_VERSION = 1
+TARGET_INDEX_MAPPING_VERSION = 2
+
+# Set an interval (in seconds) for checking if re-index and/or migration has finished
+STATUS_INTERVAL_SECONDS = 2
+
+# Set plugin names
+PLUGIN_MIGRATION_SYSTEM = 'migration-system'
+PLUGIN_APPINFO = 'appinfo-migration'
+PLUGIN_ENTITYDATA = 'collections-entity-data'
+PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
+PLUGIN_CORE_DATA = 'core-data'
+
+MANAGEMENT_APP_ID = 'b6768a08-b5d5-11e3-a495-11ddb1de66c8'
+
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Migration Tool')
+
+ parser.add_argument('--endpoint',
+ help='The endpoint to use for making API requests.',
+ type=str,
+ default='http://localhost:8080')
+
+ parser.add_argument('--super',
+ help='Superuser username and creds <user:pass>',
+ type=str,
+ required=True)
+
+ parser.add_argument('--admin',
+ help='Organization admin creds <user:pass>',
+ type=str,
+ required=True)
+
+ parser.add_argument('--init',
+ help='Init system and start first migration.',
+ action='store_true',
+ default=False)
+
+ parser.add_argument('--org',
+ help='Name of organization on which to run migration.',
+ type=str,
+ required=False)
+
+ parser.add_argument('--date',
+ help='A date from which to start the migration',
+ type=str)
+
+ parser.add_argument('--full',
+ help='Run full data migration (last step in cluster migration).',
+ action='store_true',
+ default=False)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ arg_vars = vars(my_args)
+
+ creds = arg_vars['super'].split(':')
+ if len(creds) != 2:
+ print('Superuser credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ exit_on_error()
+ else:
+ arg_vars['superuser'] = creds[0]
+ arg_vars['superpass'] = creds[1]
+
+ creds = arg_vars['super'].split(':')
+ if len(creds) != 2:
+ print('Org admin credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ exit_on_error()
+ else:
+ arg_vars['adminuser'] = creds[0]
+ arg_vars['adminpass'] = creds[1]
+
+ return arg_vars
+
+
+
+class Migrate:
+ def __init__(self):
+ self.args = parse_args()
+ self.start_date = self.args['date']
+ self.endpoint = self.args['endpoint']
+ self.metrics = {'reindex_start': '',
+ 'reindex_end': '',
+ 'appinfo_migration_start': '',
+ 'appinfo_migration_end': '',
+ 'full_data_migration_start': '',
+ 'full_data_migration_end': ''}
+ self.logger = init_logging(self.__class__.__name__)
+ self.super_user = self.args['superuser']
+ self.super_pass = self.args['superpass']
+ self.admin_user = self.args['adminuser']
+ self.admin_pass = self.args['adminpass']
+ self.org = self.args['org']
+ self.init = self.args['init']
+ self.full = self.args['full']
+
+ def run(self):
+ self.logger.info('Initializing...')
+
+ if not self.is_endpoint_available():
+ exit_on_error('Endpoint is not available, aborting')
+
+ if self.start_date is not None:
+ self.logger.info("Date Provided. Re-index will run from date=[%s]", self.start_date)
+
+ try:
+
+ if self.full:
+
+ # Do full data migration and exit
+
+ self.start_fulldata_migration()
+
+ self.metrics['full_data_migration_start'] = get_current_time()
+ self.logger.info("Full Data Migration Started")
+ is_migrated = False
+ while not is_migrated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_migrated = self.is_data_migrated()
+ if is_migrated:
+ break
+
+ self.metrics['full_data_migration_end'] = get_current_time()
+ self.logger.info("Full Data Migration completed")
+
+ self.log_metrics()
+ self.logger.info("Finished...")
+
+ return
+
+
+ if self.init:
+
+ # Init the migration system as this is the first migration done on the cluster
+
+ self.run_database_setup()
+
+ migration_system_updated = self.is_migration_system_updated()
+
+ if not migration_system_updated:
+ self.logger.info('Migration system needs to be updated. Updating migration system..')
+ self.start_migration_system_update()
+ while not migration_system_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ migration_system_updated = self.is_migration_system_updated()
+ if migration_system_updated:
+ break
+
+ index_mapping_updated = self.is_index_mapping_updated()
+
+ if not index_mapping_updated:
+ self.logger.info('Index Mapping needs to be updated. Updating index mapping..')
+ self.start_index_mapping_migration()
+ while not index_mapping_updated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ index_mapping_updated = self.is_index_mapping_updated()
+ if index_mapping_updated:
+ break
+
+
+ # Migrate app info
+
+ if self.is_appinfo_migrated():
+ self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
+ self.reset_appinfo_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+
+ self.start_appinfo_migration()
+ self.logger.info('AppInfo Migration Started.')
+ self.metrics['appinfo_migration_start'] = get_current_time()
+
+ is_appinfo_migrated = False
+ while not is_appinfo_migrated:
+ is_appinfo_migrated = self.is_appinfo_migrated()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ if is_appinfo_migrated:
+ self.metrics['appinfo_migration_end'] = get_current_time()
+ break
+ self.logger.info('AppInfo Migration Ended.')
+
+
+ # Reindex management app
+
+ job = self.start_app_reindex(MANAGEMENT_APP_ID)
+ self.metrics['reindex_start'] = get_current_time()
+ self.logger.info('Started Re-index. Job=[%s]', job)
+ is_running = True
+ while is_running:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_running = self.is_reindex_running(job)
+ if not is_running:
+ break
+
+ self.logger.info("Finished Re-index. Job=[%s]", job)
+ self.metrics['reindex_end'] = get_current_time()
+
+
+ # Dedup and re-index all of organization's apps
+
+ app_ids = self.get_app_ids()
+ for app_id in app_ids:
+
+ # De-dup app
+ job = self.start_dedup(app_id)
+ self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
+ is_running = True
+ while is_running:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_running = self.is_dedup_running(job)
+ if not is_running:
+ break
+
+ self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
+ self.metrics['dedup_end_' + app_id] = get_current_time()
+
+ # Re-index app
+ job = self.start_app_reindex(app_id)
+ self.metrics['reindex_start_' + app_id] = get_current_time()
+ self.logger.info('Started Re-index. App=[%s], Job=[%s]', app_id, job)
+ is_running = True
+ while is_running:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_running = self.is_reindex_running(job)
+ if not is_running:
+ break
+
+ self.logger.info("Finished Re-index. App=[%s], Job=[%s]", app_id, job)
+ self.metrics['reindex_end_' + app_id] = get_current_time()
+
+ self.log_metrics()
+ self.logger.info("Finished...")
+
+
+ except KeyboardInterrupt:
+ self.log_metrics()
+ self.logger.error('Keyboard interrupted migration. Please run again to ensure the migration finished.')
+
+ def get_database_setup_url(self):
+ url = self.endpoint + '/system/database/setup'
+ return url
+
+ def get_database_bootstrap_url(self):
+ url = self.endpoint + '/system/database/bootstrap'
+ return url
+
+ def get_migration_url(self):
+ url = self.endpoint + '/system/migrate/run'
+ return url
+
+ def get_reset_migration_url(self):
+ url = self.endpoint + '/system/migrate/set'
+ return url
+
+ def get_migration_status_url(self):
+ url = self.endpoint + '/system/migrate/status'
+ return url
+
+ def get_dedup_url(self):
+ url = self.endpoint + '/system/connection/dedup'
+ return url
+
+ def get_reindex_url(self):
+ url = self.endpoint + '/system/index/rebuild'
+ return url
+
+ def get_management_reindex_url(self):
+ url = self.get_reindex_url() + "/management"
+ return url
+
+ def start_core_data_migration(self):
+ try:
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def start_fulldata_migration(self):
+ try:
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def start_migration_system_update(self):
+ try:
+ # TODO fix this URL
+ migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.super_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def run_database_setup(self):
+ try:
+ setupUrl = self.get_database_setup_url()
+ r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
+ if r.status_code != 200:
+ exit_on_error('Database Setup Failed')
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to run database setup, %s', e)
+ exit_on_error(str(e))
+
+ def run_database_bootstrap(self):
+ try:
+ setupUrl = self.get_database_bootstrap_url()
+ r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
+ if r.status_code != 200:
+ exit_on_error('Database Bootstrap Failed')
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to run database bootstrap, %s', e)
+ exit_on_error(str(e))
+
+ def start_index_mapping_migration(self):
+ try:
+ migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
+ r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def start_appinfo_migration(self):
+ try:
+ migrateUrl = self.get_migration_url() + '/' + PLUGIN_APPINFO
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
+ def reset_data_migration(self):
+ version = TARGET_ENTITY_DATA_VERSION - 1
+ body = json.dumps({PLUGIN_ENTITYDATA: version, PLUGIN_APPINFO: version})
+ try:
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ self.logger.info('Resetting data migration versions to %s=[%s] '
+ 'and %s=[%s]', PLUGIN_ENTITYDATA, version, PLUGIN_APPINFO, version)
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to reset full data migration versions, %s', e)
+ exit_on_error(str(e))
+
+ def reset_appinfo_migration(self):
+ version = TARGET_APPINFO_VERSION - 1
+ body = json.dumps({PLUGIN_APPINFO: version})
+ try:
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ self.logger.info('Resetting appinfo migration versions to %s=[%s]', PLUGIN_APPINFO, version)
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to reset appinfo migration version, %s', e)
+ exit_on_error(str(e))
+
+ def is_data_migrated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ entity_version = status['data'][PLUGIN_ENTITYDATA]
+ appinfo_version = status['data'][PLUGIN_APPINFO]
+ core_data_version = status['data'][PLUGIN_CORE_DATA]
+
+ if entity_version == TARGET_ENTITY_DATA_VERSION and appinfo_version == TARGET_APPINFO_VERSION and core_data_version == TARGET_CORE_DATA_VERSION:
+ self.logger.info('Full Data Migration status=[COMPLETE], %s=[%s], '
+ '%s=[%s], %s=%s',
+ PLUGIN_ENTITYDATA,
+ entity_version,
+ PLUGIN_APPINFO,
+ appinfo_version,
+ PLUGIN_CORE_DATA,
+ core_data_version)
+ return True
+ else:
+ self.logger.info('Full Data Migration status=[NOTSTARTED/INPROGRESS]')
+ return False
+
+ def is_appinfo_migrated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ appinfo_version = status['data'][PLUGIN_APPINFO]
+
+ if appinfo_version == TARGET_APPINFO_VERSION:
+ self.logger.info('AppInfo Migration status=[COMPLETE],'
+ '%s=[%s]',
+ PLUGIN_APPINFO,
+ appinfo_version)
+ return True
+ else:
+ self.logger.info('AppInfo Migration status=[NOTSTARTED/INPROGRESS]')
+ return False
+
+ def is_migration_system_updated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ migration_system_version = status['data'][PLUGIN_MIGRATION_SYSTEM]
+
+ if migration_system_version == TARGET_MIGRATION_SYSTEM_VERSION:
+ self.logger.info('Migration System CURRENT, %s=[%s]',
+ PLUGIN_MIGRATION_SYSTEM,
+ migration_system_version)
+ return True
+ else:
+ self.logger.info('Migration System OLD, %s=[%s]',
+ PLUGIN_MIGRATION_SYSTEM,
+ migration_system_version)
+ return False
+
+ def is_index_mapping_updated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ index_mapping_version = status['data'][PLUGIN_INDEX_MAPPING]
+
+ if index_mapping_version == TARGET_INDEX_MAPPING_VERSION:
+ self.logger.info('Index Mapping CURRENT, %s=[%s]',
+ PLUGIN_INDEX_MAPPING,
+ index_mapping_version)
+ return True
+ else:
+ self.logger.info('Index Mapping OLD, %s=[%s]',
+ PLUGIN_INDEX_MAPPING,
+ index_mapping_version)
+ return False
+
+ def check_data_migration_status(self):
+
+ try:
+ r = requests.get(url=self.get_migration_status_url(), auth=(self.super_user, self.super_pass))
+ if r.status_code == 200:
+ response = r.json()
+ return response
+ else:
+ self.logger.error('Failed to check migration status, %s', r)
+ return
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to check migration status, %s', e)
+ # exit_on_error()
+
+ def get_reindex_status(self, job):
+ status_url = self.get_reindex_url()+'/' + job
+
+ try:
+ r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ return response['status']
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to get reindex status, %s', e)
+ # exit_on_error()
+
+ def start_app_reindex(self, appId):
+ body = ""
+ if self.start_date is not None:
+ body = json.dumps({'updated': self.start_date})
+
+ try:
+ r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.super_user, self.super_pass))
+
+ if r.status_code == 200:
+ response = r.json()
+ return response['jobId']
+ else:
+ self.logger.error('Failed to start reindex, %s', r)
+ exit_on_error(str(r))
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to make API request for reindex, %s', e)
+ exit_on_error(str(e))
+
+ def is_reindex_running(self, job):
+ status = self.get_reindex_status(job)
+ self.logger.info('Re-index status=[%s]', status)
+ if status != "COMPLETE":
+ return True
+ else:
+ return False
+
+ def get_dedup_status(self, job):
+ status_url = self.get_dedup_url()+'/' + job
+ try:
+ r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
+ response = r.json()
+ return response['status']['status']
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to get dedup status, %s', e)
+ # exit_on_error()
+
+ def start_dedup(self, app_id):
+ body = ""
+ try:
+ r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.super_user, self.super_pass))
+ if r.status_code == 200:
+ response = r.json()
+ return response['status']['jobStatusId']
+ else:
+ self.logger.error('Failed to start dedup, %s', r)
+ exit_on_error(str(r))
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to make API request for dedup, %s', e)
+ exit_on_error(str(e))
+
+ def is_dedup_running(self, job):
+ status = self.get_dedup_status(job)
+ self.logger.info('Dedup status=[%s]', status)
+ if status != "COMPLETE":
+ return True
+ else:
+ return False
+
+ def is_endpoint_available(self):
+
+ try:
+ r = requests.get(url=self.endpoint+'/status')
+ if r.status_code == 200:
+ return True
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Endpoint is unavailable, %s', str(e))
+ return False
+
+ def log_metrics(self):
+ self.logger.info(
+ 'Re-index start=[%s], ' +
+ 'Re-index end =[%s], ' +
+ 'Full Data Migration start=[%s], ' +
+ 'Full Data Migration end=[%s] ' +
+ 'AppInfo Migration start=[%s], ' +
+ 'AppInfo Migration end=[%s] ',
+ self.metrics['reindex_start'],
+ self.metrics['reindex_end'],
+ self.metrics['full_data_migration_start'],
+ self.metrics['full_data_migration_end'],
+ self.metrics['appinfo_migration_start'],
+ self.metrics['appinfo_migration_end']
+
+ )
+
+ def get_app_ids(self):
+
+ try:
+
+ url = self.endpoint + "/management/token"
+ body = json.dumps({"grant_type":"password","username":self.admin_user,"password":self.admin_pass})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code != 200 ):
+ print "Error logging in: " + r.text
+ return
+
+ access_token = r.json()["access_token"]
+
+ url = self.endpoint + "/management/orgs/" + self.org + "/apps?access_token=" + access_token
+ r = requests.get(url=url)
+ if r.status_code != 200:
+ exit_on_error('Cannot get app ids: ' + r.text)
+
+ apps = r.json()["data"]
+ app_ids = []
+ for appId in apps.values():
+ app_ids.append(appId)
+
+ print app_ids
+
+ return app_ids
+
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Unable to get list of application ids, %s', e)
+ exit_on_error(str(e))
+
+def get_current_time():
+ return str(int(time.time()*1000))
+
+
+def exit_on_error(e=""):
+ print ('Exiting migration script due to error: ' + str(e))
+ sys.exit(1)
+
+
+def init_logging(name):
+
+ logger = logging.getLogger(name)
+ log_file_name = './migration.log'
+ log_formatter = logging.Formatter(fmt='%(asctime)s [%(name)s] %(levelname)s %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S')
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=104857600,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+ logger.addHandler(rotating_file)
+ logger.setLevel(logging.INFO)
+
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+ logger.addHandler(stdout_logger)
+
+ return logger
+
+if __name__ == '__main__':
+
+ migration = Migrate()
+ migration.run()
[36/50] [abbrv] usergrid git commit: Updated call for new read api in
2.0
Posted by to...@apache.org.
Updated call for new read api in 2.0
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7fc7c55b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7fc7c55b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7fc7c55b
Branch: refs/heads/master
Commit: 7fc7c55b7839ff9ba5a71b1c3d74a095c27f49ff
Parents: 08e9781
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 29 13:56:14 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 13:56:14 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/rest/applications/users/UserResource.java | 4 ++--
.../java/org/apache/usergrid/management/ManagementService.java | 2 +-
.../usergrid/management/cassandra/ManagementServiceImpl.java | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index fb10245..716f367 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -169,7 +169,7 @@ public class UserResource extends ServiceResource {
@GET
@RequireSystemAccess
- @Path("password")
+ @Path("credentials")
public JSONWithPadding getUserPassword(@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
@@ -192,7 +192,7 @@ public class UserResource extends ServiceResource {
return new JSONWithPadding( response, callback );
}
- final CredentialsInfo credentialsInfo = management.getAppUserPasswordRaw( applicationId, targetUserId );
+ final CredentialsInfo credentialsInfo = management.getAppUserCredentialsInfo( applicationId, targetUserId );
response.setProperty( "credentials", credentialsInfo );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 3f02e5a..cf2924b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -288,7 +288,7 @@ public interface ManagementService {
public void setAppUserPassword( UUID applicationId, UUID userId, String oldPassword, String newPassword )
throws Exception;
- CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final UUID userId ) throws Exception;
+ CredentialsInfo getAppUserCredentialsInfo( final UUID applicationId, final UUID userId ) throws Exception;
/**
* Set the credentials info into the
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7fc7c55b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index b633727..dfd0cb1 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -2825,7 +2825,7 @@ public class ManagementServiceImpl implements ManagementService {
@Override
- public CredentialsInfo getAppUserPasswordRaw( final UUID applicationId, final UUID userId ) throws Exception {
+ public CredentialsInfo getAppUserCredentialsInfo( final UUID applicationId, final UUID userId ) throws Exception {
final User user = emf.getEntityManager( applicationId ).get( userId, User.class );
@@ -2833,7 +2833,7 @@ public class ManagementServiceImpl implements ManagementService {
throw new EntityNotFoundException("Could not find user with id " + userId + " in application" + applicationId );
}
- final CredentialsInfo ci = readUserPasswordCredentials( applicationId, userId );
+ final CredentialsInfo ci = readUserPasswordCredentials( applicationId, userId, User.ENTITY_TYPE );
if ( ci == null ) {
throw new EntityNotFoundException("Could not find credentials for user with id " + userId + " in application" + applicationId );
[44/50] [abbrv] usergrid git commit: Remove unnecessary debug print
statement.
Posted by to...@apache.org.
Remove unnecessary debug print statement.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8737f69d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8737f69d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8737f69d
Branch: refs/heads/master
Commit: 8737f69d3b78847e5ef40d0f23fec800aec0dcae
Parents: 9ada6ea
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Oct 30 08:31:00 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Oct 30 08:31:00 2015 -0400
----------------------------------------------------------------------
stack/scripts/multitenant_migrate.py | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8737f69d/stack/scripts/multitenant_migrate.py
----------------------------------------------------------------------
diff --git a/stack/scripts/multitenant_migrate.py b/stack/scripts/multitenant_migrate.py
index 62e46af..6955de7 100644
--- a/stack/scripts/multitenant_migrate.py
+++ b/stack/scripts/multitenant_migrate.py
@@ -657,8 +657,6 @@ class Migrate:
for appId in apps.values():
app_ids.append(appId)
- print app_ids
-
return app_ids
except requests.exceptions.RequestException as e:
[28/50] [abbrv] usergrid git commit: Updates the message flow to
allow for multiple processor threads per SQS take thread
Posted by to...@apache.org.
Updates the message flow to allow for multiple processor threads per SQS take thread
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/76476f17
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/76476f17
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/76476f17
Branch: refs/heads/master
Commit: 76476f17cf8e8be6f01660db3d21110eda8247f5
Parents: 1b43bda
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Oct 27 14:35:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Oct 27 14:35:34 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 68 +++++++++++---------
.../index/IndexProcessorFig.java | 13 +++-
2 files changed, 51 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/76476f17/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index d93e304..6b9abbc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -679,35 +679,45 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
while ( true );
}
- } )
- //this won't block our read loop, just reads and proceeds
- .map( messages -> {
- if ( messages == null || messages.size() == 0 ) {
- return null;
- }
-
- try {
- List<IndexEventResult> indexEventResults = callEventHandlers( messages );
- List<QueueMessage> messagesToAck = submitToIndex( indexEventResults );
- if ( messagesToAck == null || messagesToAck.size() == 0 ) {
- logger.error( "No messages came back from the queue operation should have seen "
- + messages.size(), messages );
- return messagesToAck;
- }
- if ( messagesToAck.size() < messages.size() ) {
- logger.error( "Missing messages from queue post operation", messages,
- messagesToAck );
- }
- //ack each message, but only if we didn't error.
- ack( messagesToAck );
- return messagesToAck;
- }
- catch ( Exception e ) {
- logger.error( "failed to ack messages to sqs", e );
- return null;
- //do not rethrow so we can process all of them
- }
- } );
+ } ) //this won't block our read loop, just reads and proceeds
+ .flatMap( sqsMessages -> {
+
+ //do this on a different schedule, and introduce concurrency with flatmap for faster processing
+ return Observable.just( sqsMessages )
+
+ .map( messages -> {
+ if ( messages == null || messages.size() == 0 ) {
+ return null;
+ }
+
+ try {
+ List<IndexEventResult> indexEventResults =
+ callEventHandlers( messages );
+ List<QueueMessage> messagesToAck =
+ submitToIndex( indexEventResults );
+ if ( messagesToAck == null || messagesToAck.size() == 0 ) {
+ logger.error(
+ "No messages came back from the queue operation should "
+ + "have seen "
+ + messages.size(), messages );
+ return messagesToAck;
+ }
+ if ( messagesToAck.size() < messages.size() ) {
+ logger.error( "Missing messages from queue post operation",
+ messages, messagesToAck );
+ }
+ //ack each message, but only if we didn't error.
+ ack( messagesToAck );
+ return messagesToAck;
+ }
+ catch ( Exception e ) {
+ logger.error( "failed to ack messages to sqs", e );
+ return null;
+ //do not rethrow so we can process all of them
+ }
+ } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() );
+ //end flatMap
+ }, indexProcessorFig.getEventConcurrencyFactor() );
//start in the background
http://git-wip-us.apache.org/repos/asf/usergrid/blob/76476f17/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 7650c62..9d02717 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -36,6 +36,8 @@ public interface IndexProcessorFig extends GuicyFig {
String ELASTICSEARCH_WORKER_COUNT = "elasticsearch.worker_count";
+ String EVENT_CONCURRENCY_FACTOR = "event.concurrency.factor";
+
String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
String INDEX_QUEUE_READ_TIMEOUT = "elasticsearch.queue_read_timeout";
@@ -70,9 +72,18 @@ public interface IndexProcessorFig extends GuicyFig {
int getIndexQueueVisibilityTimeout();
/**
+ * The number of worker threads used when handing off messages from the SQS thread
+ */
+ @Default( "20" )
+ @Key( EVENT_CONCURRENCY_FACTOR )
+ int getEventConcurrencyFactor();
+
+
+
+ /**
* The number of worker threads used to read index write requests from the queue.
*/
- @Default( "16" )
+ @Default( "8" )
@Key( ELASTICSEARCH_WORKER_COUNT )
int getWorkerCount();
[31/50] [abbrv] usergrid git commit: Fix failing test to ensure we
don't set the same timestamp for all entities inserted into the index (since
we don't do actually do that). Also ensure it's a long as what's required of
the timestamps
Posted by to...@apache.org.
Fix failing test to ensure we don't set the same timestamp for all entities inserted into the index (since we don't do actually do that). Also ensure it's a long as what's required of the timestamps
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c652171f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c652171f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c652171f
Branch: refs/heads/master
Commit: c652171f476f6ad953c94328e0fb2f4ea0b3aa44
Parents: 22beca2
Author: Michael Russo <mi...@gmail.com>
Authored: Wed Oct 28 16:56:19 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Wed Oct 28 16:56:19 2015 -0700
----------------------------------------------------------------------
.../persistence/index/impl/EntityIndexTest.java | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c652171f/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 5243d5a..d6758dd 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicLong;
import com.google.common.base.Optional;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.model.field.*;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -48,10 +49,6 @@ import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.IntegerField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.util.EntityUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -394,7 +391,6 @@ public class EntityIndexTest extends BaseIT {
int numberOfEntities = 1000;
int versionToSearchFor = numberOfEntities / 2;
- IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, 1 );
UUID entityUUID = UUID.randomUUID();
Id entityId = new SimpleId( "mehCar" );
@@ -408,9 +404,11 @@ public class EntityIndexTest extends BaseIT {
Entity[] entity = new Entity[numberOfEntities];
for(int i = 0; i < numberOfEntities; i++) {
entity[i] = EntityIndexMapUtils.fromMap( entityMap );
- EntityUtils.setId( entity[i], entityId );
- EntityUtils.setVersion( entity[i], UUIDGenerator.newTimeUUID() );
- entity[i].setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID ) );
+ EntityUtils.setId(entity[i], entityId);
+ EntityUtils.setVersion(entity[i], UUIDGenerator.newTimeUUID());
+ entity[i].setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID));
+
+ IndexEdge searchEdge = new IndexEdgeImpl( appId, "mehCars", SearchEdge.NodeType.SOURCE, System.currentTimeMillis()*1000 );
//index the new entity. This is where the loop will be set to create like 100 entities.
indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i] ).build()).subscribe();
[20/50] [abbrv] usergrid git commit: First pass. Need to test.
Posted by to...@apache.org.
First pass. Need to test.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e41b5f02
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e41b5f02
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e41b5f02
Branch: refs/heads/master
Commit: e41b5f02eb12f72b9c19e5ae079b32d528951b47
Parents: 07d2ad3
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Oct 26 16:24:37 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Oct 26 16:24:37 2015 -0600
----------------------------------------------------------------------
.../service/StatusServiceImpl.java | 3 +-
.../usergrid/rest/ConnectionResource.java | 198 +++++++++++++++++++
.../apache/usergrid/rest/SystemResource.java | 5 +
3 files changed, 205 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e41b5f02/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
index 93fe653..282929e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/StatusServiceImpl.java
@@ -93,8 +93,9 @@ public class StatusServiceImpl implements StatusService {
final MapManager mapManager = mapManagerFactory.createMapManager(new MapScopeImpl(appId, "status"));
try {
String statusVal = mapManager.getString(jobString + statusKey);
+ //nothing to emit
if(statusVal==null){
- subscriber.onNext(null);
+ subscriber.onCompleted();
}else {
final Map<String, Object> data = MAPPER.readValue(mapManager.getString(jobString + dataKey), Map.class);
final Status status = Status.valueOf(statusVal);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e41b5f02/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
new file mode 100644
index 0000000..b6a38e1
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.rest;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.json.JSONWithPadding;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+
+/**
+ * system/index/otherstuff
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class ConnectionResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( ConnectionResource.class );
+
+ public ConnectionResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "dedup/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding all applications" );
+
+ final UUID applicationId = UUIDUtils.tryGetUUID( applicationIdStr );
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+
+ return executeAndCreateResponse( applicationId, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "dedup/{jobId}" )
+ public JSONWithPadding rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+ logger.info( "Getting status for index jobs" );
+
+ Preconditions.checkNotNull( jobId, "query param jobId must not be null" );
+
+
+ final UUID jobUUID = UUIDUtils.tryGetUUID( jobId );
+
+ final StatusService.JobStatus
+ job = getStatusService().getStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobUUID ).toBlocking().lastOrDefault(
+ null );
+
+ Preconditions.checkNotNull( job, "job with id '" + jobId + "' does not exist" );
+
+
+ return createResult( job, callback );
+ }
+
+
+ private ConnectionService getConnectionService() {
+ return injector.getInstance( ConnectionServiceImpl.class );
+ }
+
+
+ private StatusService getStatusService() {
+ return injector.getInstance( StatusService.class );
+ }
+
+
+
+ /**
+ * Execute the request and return the response.
+ */
+ private JSONWithPadding executeAndCreateResponse( final UUID applicationId, final String callback ) {
+
+ final Observable<ApplicationScope> applicationScopeObservable =
+ Observable.just( CpNamingUtils.getApplicationScope( applicationId ) );
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final StatusService statusService = getStatusService();
+ final ConnectionService connectionService = getConnectionService();
+
+ final AtomicLong count = new AtomicLong( 0 );
+
+ //start de duping and run in the background
+ connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
+ .doOnNext( buffer -> {
+
+
+ final long runningTotal = count.addAndGet( buffer.size() );
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put( "countProcessed", runningTotal );
+ put( "updatedTimestamp", System.currentTimeMillis() );
+ }};
+
+ statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
+ StatusService.Status.INPROGRESS, status );
+ } ).doOnSubscribe( () -> {
+ statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.STARTED,
+ new HashMap<>() );
+ } ).doOnCompleted( () -> {
+
+ final long runningTotal = count.get();
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put( "countProcessed", runningTotal );
+ put( "updatedTimestamp", System.currentTimeMillis() );
+ }};
+
+ statusService
+ .setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.COMPLETE, status );
+ } ).subscribeOn( Schedulers.newThread() ).subscribe();
+
+
+ final StatusService.JobStatus status = new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
+ return createResult( status, callback );
+ }
+
+
+ /**
+ * Create a response with the specified data.
+ * @param jobStatus
+ * @param callback
+ * @return
+ */
+ private JSONWithPadding createResult(final StatusService.JobStatus jobStatus, final String callback){
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "de-dup connections" );
+ response.setProperty( "status", jobStatus );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e41b5f02/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
index f266441..aaee596 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
@@ -100,4 +100,9 @@ public class SystemResource extends AbstractContextResource {
public ApplicationsResource applications() {
return getSubResource( ApplicationsResource.class );
}
+
+
+ @Path( "connection" )
+ public ConnectionResource connection() { return getSubResource( ConnectionResource.class ); }
+
}
[34/50] [abbrv] usergrid git commit: Adds the ability to PUT
credentials as a superuser
Posted by to...@apache.org.
Adds the ability to PUT credentials as a superuser
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5ed8c7ce
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5ed8c7ce
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5ed8c7ce
Branch: refs/heads/master
Commit: 5ed8c7ce14d253ba04dfad48d239a0e11bf1a33c
Parents: c652171
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 29 12:36:21 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 29 13:47:15 2015 -0600
----------------------------------------------------------------------
.../rest/applications/users/UserResource.java | 38 ++++++++++++++++++++
.../usergrid/management/ManagementService.java | 12 ++++++-
.../cassandra/ManagementServiceImpl.java | 21 ++++++++++-
3 files changed, 69 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ed8c7ce/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index a8b0f81..df88cf0 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.apache.usergrid.management.ActivationState;
+import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.entities.User;
@@ -165,6 +166,43 @@ public class UserResource extends ServiceResource {
}
+ @PUT
+ @Path("credentials")
+ public JSONWithPadding setUserCredentials( @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ logger.info( "UserResource.setUserPassword" );
+
+ if ( json == null ) {
+ return null;
+ }
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "set user credentials" );
+ Object credentials = json.get( "credentials" );
+
+
+ if ( credentials == null ) {
+ throw new IllegalArgumentException( "credentials sub object is required" );
+ }
+
+ UUID applicationId = getApplicationId();
+ UUID targetUserId = getUserUuid();
+
+ if ( targetUserId == null ) {
+ response.setError( "User not found" );
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ management.setAppUserCredentialsInfo( applicationId, targetUserId, ( CredentialsInfo ) credentials );
+
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
@POST
@Path("password")
public JSONWithPadding setUserPasswordPost( @Context UriInfo ui, Map<String, Object> json,
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ed8c7ce/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 3c5bbdb..d69de2e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -288,6 +288,16 @@ public interface ManagementService {
public void setAppUserPassword( UUID applicationId, UUID userId, String oldPassword, String newPassword )
throws Exception;
+ /**
+ * Set the credentials info into the
+ * @param applicationId
+ * @param userId
+ * @param credentialsInfo
+ * @throws Exception
+ */
+ void setAppUserCredentialsInfo( final UUID applicationId, final UUID userId, final CredentialsInfo credentialsInfo ) throws Exception;
+
+
public User verifyAppUserPasswordCredentials( UUID applicationId, String name, String password ) throws Exception;
public UserInfo getAppUserFromAccessToken( String token ) throws Exception;
@@ -350,7 +360,7 @@ public interface ManagementService {
public OrganizationConfig getOrganizationConfigForApplication( UUID applicationId ) throws Exception;
public void updateOrganizationConfig( OrganizationConfig organizationConfig ) throws Exception;
-
+
/**
* will delete all entities
* @param applicationId
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ed8c7ce/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 70d74fc..2e33539 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.management.cassandra;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import com.google.inject.Injector;
@@ -2806,9 +2807,27 @@ public class ManagementServiceImpl implements ManagementService {
@Override
+ public void setAppUserCredentialsInfo( final UUID applicationId, final UUID userId,
+ final CredentialsInfo credentialsInfo ) throws Exception {
+
+ Preconditions.checkNotNull( applicationId, "applicationId is required" );
+ Preconditions.checkNotNull( userId, "userId is required" );
+ Preconditions.checkNotNull( credentialsInfo, "credentialsInfo is required" );
+
+ final User user = emf.getEntityManager( applicationId ).get(userId, User.class);
+
+ if(user == null){
+ throw new EntityNotFoundException( "User with id " + userId + " cannot be found" );
+ }
+
+ writeUserPassword(applicationId, user, credentialsInfo);
+ }
+
+
+ @Override
public User verifyAppUserPasswordCredentials( UUID applicationId, String name, String password ) throws Exception {
- User user = findUserEntity(applicationId, name);
+ User user = findUserEntity( applicationId, name );
if ( user == null ) {
return null;
}
[15/50] [abbrv] usergrid git commit: Cleaned up unused import
Posted by to...@apache.org.
Cleaned up unused import
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6e34b20b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6e34b20b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6e34b20b
Branch: refs/heads/master
Commit: 6e34b20b7e1e0d125c7a3d787334d8c1293487a3
Parents: a770a6d
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 23 16:50:43 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 23 16:50:43 2015 -0600
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6e34b20b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
index 88809e2..54d1596 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.persistence.graph;
-import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
[33/50] [abbrv] usergrid git commit: Added --super and --admin so we
can specify both superuser and org admin credentials. Did some testing and
migration seems to work for one tenant,
have not tried two yet. Commented out the de-dup calls because de-dup
Posted by to...@apache.org.
Added --super and --admin so we can specify both superuser and org admin credentials.
Did some testing and migration seems to work for one tenant, have not tried two yet.
Commented out the de-dup calls because de-dup never finishes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/471dc359
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/471dc359
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/471dc359
Branch: refs/heads/master
Commit: 471dc359d5920ea521c4058e7829a37494a17041
Parents: d79bf4c
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 10:07:33 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 10:07:33 2015 -0400
----------------------------------------------------------------------
stack/scripts/migrate_entity_data.py | 130 ++++++++++++++++++++----------
1 file changed, 87 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/471dc359/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index f576108..0edd319 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -20,7 +20,9 @@
#
# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
#
-# python migrate_entity_data.py --org <org1name> --user <superuser>:<superpass> --init
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+#
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --init
#
# This command will setup and bootstrap the database, setup the migration system and update index mappings:
# - /system/database/setup
@@ -31,24 +33,34 @@
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app.
#
+# Write down the 'Re-index start' timestamp when this is finished.
+#
# STEP 2 - PUT TENANT ONE TOMCATS IN SERVICE AND DO DELTA MIGRATION
#
-# python migrate_entity_data.py --org <org1name> --user <superuser>:<superpass> --date
+# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
+#
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
#
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app with a start-date specified so only data modified since
# STEP 1 will be re-indexed.
#
-# STEP 3 - SETUP TENENT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
+#
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
#
-# python migrate_entity_data.py --org <org2name> --user <superuser>:<superpass> --date
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass>
#
# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app.
#
+# Write down the 'Re-index start' timestamp when this is finished.
+
# STEP 4 - PUT TENANT TWO TOMCATS IN SERVICE AND DO DELTA MIGRATION
#
-# python migrate_entity_data.py --org <org2name> --user <superuser>:<superpass> --date
+# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
+#
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
#
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app with a start-date specified so only data modified since
@@ -56,7 +68,9 @@
#
# STEP 5 - FULL DATA MIGRATION
#
-# python migrate_entity_data.py --user <superuser>:<superpass> --full
+# Login to any Tomcat instance in the cluster and run this command (admin user creds must be specificed but will be ignored):
+#
+# python migrate_entity_data.py --super <user>:<pass> --admin <user>:<pass> --full
#
# This command will run the full data migration.
#
@@ -87,6 +101,8 @@ PLUGIN_ENTITYDATA = 'collections-entity-data'
PLUGIN_INDEX_MAPPING = 'index_mapping_migration'
PLUGIN_CORE_DATA = 'core-data'
+MANAGEMENT_APP_ID = 'b6768a08-b5d5-11e3-a495-11ddb1de66c8'
+
def parse_args():
@@ -97,8 +113,13 @@ def parse_args():
type=str,
default='http://localhost:8080')
- parser.add_argument('--user',
- help='System Admin Credentials used to authenticate with Usergrid <user:pass>',
+ parser.add_argument('--super',
+ help='Superuser username and creds <user:pass>',
+ type=str,
+ required=True)
+
+ parser.add_argument('--admin',
+ help='Organization admin creds <user:pass>',
type=str,
required=True)
@@ -124,13 +145,22 @@ def parse_args():
my_args = parser.parse_args(sys.argv[1:])
arg_vars = vars(my_args)
- creds = arg_vars['user'].split(':')
+
+ creds = arg_vars['super'].split(':')
+ if len(creds) != 2:
+ print('Superuser credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ exit_on_error()
+ else:
+ arg_vars['superuser'] = creds[0]
+ arg_vars['superpass'] = creds[1]
+
+ creds = arg_vars['super'].split(':')
if len(creds) != 2:
- print('Credentials not properly specified. Must be "-u <user:pass>". Exiting...')
+ print('Org admin credentials not properly specified. Must be "-u <user:pass>". Exiting...')
exit_on_error()
else:
- arg_vars['user'] = creds[0]
- arg_vars['pass'] = creds[1]
+ arg_vars['adminuser'] = creds[0]
+ arg_vars['adminpass'] = creds[1]
return arg_vars
@@ -148,8 +178,10 @@ class Migrate:
'full_data_migration_start': '',
'full_data_migration_end': ''}
self.logger = init_logging(self.__class__.__name__)
- self.admin_user = self.args['user']
- self.admin_pass = self.args['pass']
+ self.super_user = self.args['superuser']
+ self.super_pass = self.args['superpass']
+ self.admin_user = self.args['adminuser']
+ self.admin_pass = self.args['adminpass']
self.org = self.args['org']
self.init = self.args['init']
self.full = self.args['full']
@@ -241,7 +273,7 @@ class Migrate:
# Reindex management app
- job = self.start_reindex()
+ job = self.start_app_reindex(MANAGEMENT_APP_ID)
self.metrics['reindex_start'] = get_current_time()
self.logger.info('Started Re-index. Job=[%s]', job)
is_running = True
@@ -261,18 +293,18 @@ class Migrate:
for app_id in app_ids:
# De-dep app
- job = self.start_app_dedup(app_id)
- self.metrics['dedup_start_' + app_id] = get_current_time()
- self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
- is_running = True
- while is_running:
- time.sleep(STATUS_INTERVAL_SECONDS)
- is_running = self.is_reindex_running(job)
- if not is_running:
- break
-
- self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
- self.metrics['dedup_end_' + app_id] = get_current_time()
+ # job = self.start_dedup(app_id)
+ # self.metrics['dedup_start_' + app_id] = get_current_time()
+ # self.logger.info('Started dedup. App=[%s], Job=[%s]', app_id, job)
+ # is_running = True
+ # while is_running:
+ # time.sleep(STATUS_INTERVAL_SECONDS)
+ # is_running = self.is_dedup_running(job)
+ # if not is_running:
+ # break
+ #
+ # self.logger.info("Finished dedup. App=[%s], Job=[%s]", app_id, job)
+ # self.metrics['dedup_end_' + app_id] = get_current_time()
# Re-index app
job = self.start_app_reindex(app_id)
@@ -330,7 +362,7 @@ class Migrate:
def start_core_data_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -339,7 +371,7 @@ class Migrate:
def start_fulldata_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -350,7 +382,7 @@ class Migrate:
try:
# TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -360,7 +392,7 @@ class Migrate:
def run_database_setup(self):
try:
setupUrl = self.get_database_setup_url()
- r = requests.put(url=setupUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
if r.status_code != 200:
exit_on_error('Database Setup Failed')
@@ -371,7 +403,7 @@ class Migrate:
def run_database_bootstrap(self):
try:
setupUrl = self.get_database_bootstrap_url()
- r = requests.put(url=setupUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=setupUrl, auth=(self.super_user, self.super_pass))
if r.status_code != 200:
exit_on_error('Database Bootstrap Failed')
@@ -382,7 +414,7 @@ class Migrate:
def start_index_mapping_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_INDEX_MAPPING
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -403,7 +435,7 @@ class Migrate:
version = TARGET_ENTITY_DATA_VERSION - 1
body = json.dumps({PLUGIN_ENTITYDATA: version, PLUGIN_APPINFO: version})
try:
- r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
response = r.json()
self.logger.info('Resetting data migration versions to %s=[%s] '
'and %s=[%s]', PLUGIN_ENTITYDATA, version, PLUGIN_APPINFO, version)
@@ -416,7 +448,7 @@ class Migrate:
version = TARGET_APPINFO_VERSION - 1
body = json.dumps({PLUGIN_APPINFO: version})
try:
- r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.super_user, self.super_pass))
response = r.json()
self.logger.info('Resetting appinfo migration versions to %s=[%s]', PLUGIN_APPINFO, version)
return response
@@ -495,7 +527,7 @@ class Migrate:
def check_data_migration_status(self):
try:
- r = requests.get(url=self.get_migration_status_url(), auth=(self.admin_user, self.admin_pass))
+ r = requests.get(url=self.get_migration_status_url(), auth=(self.super_user, self.super_pass))
if r.status_code == 200:
response = r.json()
return response
@@ -510,7 +542,7 @@ class Migrate:
status_url = self.get_reindex_url()+'/' + job
try:
- r = requests.get(url=status_url, auth=(self.admin_user, self.admin_pass))
+ r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
response = r.json()
return response['status']
except requests.exceptions.RequestException as e:
@@ -523,7 +555,7 @@ class Migrate:
body = json.dumps({'updated': self.start_date})
try:
- r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.admin_user, self.admin_pass))
+ r = requests.post(url=self.get_reindex_url(), data=body, auth=(self.super_user, self.super_pass))
if r.status_code == 200:
response = r.json()
@@ -546,7 +578,8 @@ class Migrate:
def get_dedup_status(self, job):
status_url = self.get_dedup_url()+'/' + job
try:
- r = requests.get(url=status_url, auth=(self.admin_user, self.admin_pass))
+ r = requests.get(url=status_url, auth=(self.super_user, self.super_pass))
+ print r.text
response = r.json()
return response['status']
except requests.exceptions.RequestException as e:
@@ -556,10 +589,11 @@ class Migrate:
def start_dedup(self, app_id):
body = ""
try:
- r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.admin_user, self.admin_pass))
+ r = requests.post(url=self.get_dedup_url() + "/" + app_id, data=body, auth=(self.super_user, self.super_pass))
if r.status_code == 200:
response = r.json()
- return response['jobId']
+ print r.text
+ return response['status']['jobStatusId']
else:
self.logger.error('Failed to start dedup, %s', r)
exit_on_error(str(r))
@@ -606,10 +640,20 @@ class Migrate:
def get_app_ids(self):
try:
- url = self.endpoint + "/management/orgs/" + self.org + "/apps"
+
+ url = self.endpoint + "/management/token"
+ body = json.dumps({"grant_type":"password","username":self.admin_user,"password":self.admin_pass})
+ r = requests.post(url=url, data=body)
+ if ( r.status_code != 200 ):
+ print "Error logging in: " + r.text
+ return
+
+ access_token = r.json()["access_token"]
+
+ url = self.endpoint + "/management/orgs/" + self.org + "/apps?access_token=" + access_token
r = requests.get(url=url)
if r.status_code != 200:
- exit_on_error('Database Bootstrap Failed')
+ exit_on_error('Cannot get app ids: ' + r.text)
apps = r.json()["data"]
app_ids = []
[02/50] [abbrv] usergrid git commit: Fix issues with integrating with
Apigee's mobile analytics.
Posted by to...@apache.org.
Fix issues with integrating with Apigee's mobile analytics.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c9bf87c6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c9bf87c6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c9bf87c6
Branch: refs/heads/master
Commit: c9bf87c68defd1cb84bd8b65220b1994f0e8b17e
Parents: 88d1179
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Oct 22 10:20:42 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Oct 22 10:20:42 2015 -0700
----------------------------------------------------------------------
.../usergrid/management/cassandra/ManagementServiceImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c9bf87c6/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 03b798c..70d74fc 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -1610,7 +1610,8 @@ public class ManagementServiceImpl implements ManagementService {
@Override
public ApplicationInfo createApplication( UUID organizationId, String applicationName ) throws Exception {
- return createApplication( organizationId, applicationName,null, null );
+ // DO NOT CHANGE THIS AS SOME EXTERNAL CLASSES MAY RELY ON THIS BEHAVIOR WHEN EXTENDING
+ return createApplication( organizationId, applicationName, null );
}
[05/50] [abbrv] usergrid git commit: Merge branch 'USERGRID-1052' of
https://git-wip-us.apache.org/repos/asf/usergrid into USERGRID-1052
Posted by to...@apache.org.
Merge branch 'USERGRID-1052' of https://git-wip-us.apache.org/repos/asf/usergrid into USERGRID-1052
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/48161a1a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/48161a1a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/48161a1a
Branch: refs/heads/master
Commit: 48161a1ac9bfbf3e8c3aefa9586fc104ff0ec664
Parents: b1e1ed1 1b83a86
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Oct 22 12:13:08 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Oct 22 12:13:08 2015 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/rest/applications/ApplicationResource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[23/50] [abbrv] usergrid git commit: Make the graph read repair
directly compact nodes in graph instead of queueing events. Misc prop file
changes.
Posted by to...@apache.org.
Make the graph read repair directly compact nodes in graph instead of queueing events. Misc prop file changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/70d7a958
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/70d7a958
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/70d7a958
Branch: refs/heads/master
Commit: 70d7a9586ece0f32ec5aa50334cd4d70f440b2c6
Parents: 5eed978
Author: Michael Russo <mi...@gmail.com>
Authored: Mon Oct 26 22:26:07 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Mon Oct 26 22:26:07 2015 -0700
----------------------------------------------------------------------
.../index/IndexProcessorFig.java | 4 ++--
.../read/traverse/AbstractReadGraphFilter.java | 25 ++++++++++----------
.../traverse/ReadGraphCollectionFilter.java | 6 ++---
.../traverse/ReadGraphConnectionFilter.java | 6 ++---
4 files changed, 21 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70d7a958/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index ec9b315..7650c62 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -65,14 +65,14 @@ public interface IndexProcessorFig extends GuicyFig {
* Received messages will remain 'in flight' until they are ack'd(deleted) or this timeout occurs.
* If the timeout occurs, the messages will become visible again for re-processing.
*/
- @Default( "5000" ) // 5 seconds
+ @Default( "30000" ) // 30 seconds
@Key( INDEX_QUEUE_VISIBILITY_TIMEOUT )
int getIndexQueueVisibilityTimeout();
/**
* The number of worker threads used to read index write requests from the queue.
*/
- @Default( "8" )
+ @Default( "16" )
@Key( ELASTICSEARCH_WORKER_COUNT )
int getWorkerCount();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70d7a958/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 621edd2..9d050c8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,10 +20,10 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
@@ -50,16 +50,16 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
private static final Logger logger = LoggerFactory.getLogger( AbstractReadGraphFilter.class );
private final GraphManagerFactory graphManagerFactory;
- private final AsyncEventService asyncEventService;
+ private final RxTaskScheduler rxTaskScheduler;
/**
* Create a new instance of our command
*/
public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory,
- final AsyncEventService asyncEventService ) {
+ final RxTaskScheduler rxTaskScheduler) {
this.graphManagerFactory = graphManagerFactory;
- this.asyncEventService = asyncEventService;
+ this.rxTaskScheduler = rxTaskScheduler;
}
@@ -109,25 +109,26 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
if(isDeleted){
- logger.trace( "Edge {} is deleted, queueing the delete event", markedEdge );
- asyncEventService.queueDeleteEdge( applicationScope, markedEdge );
+ logger.trace("Edge {} is deleted, deleting the edge", markedEdge);
+ graphManager.deleteEdge(markedEdge).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ .subscribe();
}
if(isSourceNodeDeleted){
final Id sourceNodeId = markedEdge.getSourceNode();
- logger.trace( "Edge {} has a deleted source node, queueing the delete entity event for id {}", markedEdge, sourceNodeId );
-
- asyncEventService.queueEntityDelete( applicationScope, sourceNodeId );
+ logger.trace("Edge {} has a deleted source node, deleting the entity for id {}", markedEdge, sourceNodeId);
+ graphManager.compactNode(sourceNodeId).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ .subscribe();
}
if(isTargetNodeDelete){
final Id targetNodeId = markedEdge.getTargetNode();
- logger.trace( "Edge {} has a deleted target node, queueing the delete entity event for id {}", markedEdge, targetNodeId );
-
- asyncEventService.queueEntityDelete( applicationScope, targetNodeId );
+ logger.trace("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId );
+ graphManager.compactNode(targetNodeId).subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
+ .subscribe();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70d7a958/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
index db5a0a8..1d63bc6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
-import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import com.google.inject.Inject;
@@ -41,8 +41,8 @@ public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, final AsyncEventService asyncEventService, @Assisted final String collectionName ) {
- super( graphManagerFactory, asyncEventService );
+ public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, final RxTaskScheduler rxTaskScheduler, @Assisted final String collectionName ) {
+ super( graphManagerFactory, rxTaskScheduler );
this.collectionName = collectionName;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70d7a958/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
index 93e8fd4..efe94db 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
-import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import com.google.inject.Inject;
@@ -41,8 +41,8 @@ public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
* Create a new instance of our command
*/
@Inject
- public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, final AsyncEventService asyncEventService, @Assisted final String connectionName ) {
- super( graphManagerFactory, asyncEventService );
+ public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, final RxTaskScheduler rxTaskScheduler, @Assisted final String connectionName ) {
+ super( graphManagerFactory, rxTaskScheduler );
this.connectionName = connectionName;
}
[16/50] [abbrv] usergrid git commit: Since delete event is also being
used for read repair,
there are possibilities where the ES documents are already removed,
but entities and edges need cleaned from c*. Don't validate empty sets
returned from processi
Posted by to...@apache.org.
Since delete event is also being used for read repair, there are possibilities where the ES documents are already removed, but entities and edges need cleaned from c*. Don't validate empty sets returned from processing the event as it's a possible happy path case.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b025dda9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b025dda9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b025dda9
Branch: refs/heads/master
Commit: b025dda98f97b964df9b63a5622aa3a735d2813a
Parents: 6e34b20
Author: Michael Russo <mi...@gmail.com>
Authored: Sat Oct 24 20:15:21 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Sat Oct 24 20:15:21 2015 -0700
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b025dda9/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index fe53776..d93e304 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -324,6 +324,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
else if ( event instanceof EntityDeleteEvent ) {
indexoperationObservable = handleEntityDelete( message );
+ validateEmptySets = false; // do not check this one for an empty set b/c it can be empty
+
}
else if ( event instanceof EntityIndexEvent ) {
indexoperationObservable = handleEntityIndexUpdate( message );
[25/50] [abbrv] usergrid git commit: Moves system endpoints into the
system package
Posted by to...@apache.org.
Moves system endpoints into the system package
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7a4f36ab
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7a4f36ab
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7a4f36ab
Branch: refs/heads/master
Commit: 7a4f36ab9c0e504133f80f4d0ad8a70d115ee4f4
Parents: 3b43624
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Oct 27 09:46:06 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Oct 27 09:46:06 2015 -0600
----------------------------------------------------------------------
.../usergrid/rest/ApplicationsResource.java | 178 ----------
.../usergrid/rest/ConnectionResource.java | 199 -----------
.../apache/usergrid/rest/DatabaseResource.java | 98 ------
.../org/apache/usergrid/rest/IndexResource.java | 328 ------------------
.../apache/usergrid/rest/MigrateResource.java | 268 ---------------
.../apache/usergrid/rest/SystemResource.java | 108 ------
.../rest/system/ApplicationsResource.java | 178 ++++++++++
.../rest/system/ConnectionResource.java | 202 +++++++++++
.../usergrid/rest/system/DatabaseResource.java | 100 ++++++
.../usergrid/rest/system/IndexResource.java | 331 +++++++++++++++++++
.../usergrid/rest/system/MigrateResource.java | 270 +++++++++++++++
.../usergrid/rest/system/SystemResource.java | 108 ++++++
12 files changed, 1189 insertions(+), 1179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
deleted file mode 100644
index 55cf0f6..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ApplicationsResource.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-package org.apache.usergrid.rest;
-
-import com.sun.jersey.api.json.JSONWithPadding;
-import org.apache.usergrid.corepersistence.service.StatusService;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.core.util.StringUtils;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Classy class class.
- */
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class ApplicationsResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger(ApplicationsResource.class);
-
-
- public ApplicationsResource() {
-
- logger.info( "ApplicationsResource initialized" );
- } {
-
- }
-
- @RequireSystemAccess
- @DELETE
- @Path( "{applicationId}" )
- public JSONWithPadding clearApplication( @Context UriInfo ui,
- @PathParam("applicationId") UUID applicationId,
- @QueryParam( "confirmApplicationName" ) String confirmApplicationName,
- @QueryParam( "limit" ) int limit,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- if(confirmApplicationName == null){
- throw new IllegalArgumentException("please make add a QueryString for confirmApplicationName");
- }
-
- final UUID jobId = UUIDGenerator.newTimeUUID();
-
- final EntityManager em = emf.getEntityManager(applicationId);
- final String name = em.getApplication().getApplicationName();
- if(!name.toLowerCase().equals(confirmApplicationName.toLowerCase())){
- throw new IllegalArgumentException("confirmApplicationName: " + confirmApplicationName + " does not equal " + name);
- }
- final StatusService statusService = injector.getInstance(StatusService.class);
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "clear application" );
-
- logger.info("clearing up application");
-
-
- final Thread delete = new Thread() {
-
- @Override
- public void run() {
- final AtomicInteger itemsDeleted = new AtomicInteger(0);
- try {
- management.deleteAllEntities(applicationId, limit)
- .map(id -> itemsDeleted.incrementAndGet())
- .doOnNext(count -> {
- if( count % 100 == 0 ){
- Map<String,Object> map = new LinkedHashMap<>();
- map.put("count",itemsDeleted.intValue());
- final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map)
- .subscribe();//do not want to throw this exception
- }
- })
- .doOnCompleted(() -> {
- Map<String, Object> map = new LinkedHashMap<>();
- map.put("count", itemsDeleted.intValue());
- final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId, jobId, StatusService.Status.COMPLETE, map)
- .toBlocking().lastOrDefault(null);//want to rethrow this exception
- })
- .toBlocking().lastOrDefault(null);//expecting exception to be caught if job fails
-
- } catch ( Exception e ) {
- Map<String,Object> map = new LinkedHashMap<>();
- map.put("exception",e);
- try {
- statusService.setStatus(applicationId, jobId, StatusService.Status.FAILED, map).toBlocking().lastOrDefault(null);//leave as subscribe if fails retry
- }catch (Exception subE){
- logger.error("failed to update status "+jobId,subE);
- }
- logger.error( "Failed to delete appid:"+applicationId + " jobid:"+jobId+" count:"+itemsDeleted, e );
- }
- }
- };
-
- delete.setName("Delete for app : " + applicationId + " job: " + jobId);
- delete.setDaemon(true);
- delete.start();
-
- try {
- //should throw exception if can't start
- statusService.setStatus(applicationId, jobId, StatusService.Status.STARTED, new LinkedHashMap<>()).toBlocking().lastOrDefault(null);
- }catch (Exception e){
- logger.error("failed to set status for " + jobId, e);
- }
- Map<String,Object> data = new HashMap<>();
- data.put("jobId",jobId);
- data.put("status",StatusService.Status.STARTED);
- response.setData(data);
- response.setSuccess();
- return new JSONWithPadding( response, callback );
- }
-
- @RequireSystemAccess
- @GET
- @Path( "{applicationId}/job/{jobId}" )
- public JSONWithPadding getStatus( @Context UriInfo ui,
- @PathParam("applicationId") UUID applicationId,
- @PathParam("jobId") UUID jobId,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception{
- final StatusService statusService = injector.getInstance(StatusService.class);
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "clear application" );
-
- StatusService.JobStatus jobStatus = statusService.getStatus(applicationId, jobId).toBlocking().lastOrDefault(null);
-
- Map<String,Object> data = new HashMap<>();
- data.put("jobId",jobId);
- data.put( "status", jobStatus.getStatus().toString() );
- data.put( "metadata", jobStatus.getData() );
- response.setData(data);
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
deleted file mode 100644
index dfa4781..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.usergrid.rest;
-
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import org.apache.usergrid.corepersistence.service.ConnectionService;
-import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
-import org.apache.usergrid.corepersistence.service.StatusService;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.query.Identifier;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
-
-import rx.Observable;
-import rx.schedulers.Schedulers;
-
-
-/**
- * system/index/otherstuff
- */
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class ConnectionResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( ConnectionResource.class );
-
- public ConnectionResource() {
- super();
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "dedup/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
-
- logger.info( "Rebuilding all applications" );
-
- final UUID applicationId = UUIDUtils.tryGetUUID( applicationIdStr );
-
- Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
-
- return executeAndCreateResponse( applicationId, callback );
- }
-
-
- @RequireSystemAccess
- @GET
- @Path( "dedup/{jobId: " + Identifier.UUID_REX + "}" )
- public JSONWithPadding rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
- logger.info( "Getting status for index jobs" );
-
- Preconditions.checkNotNull( jobId, "query param jobId must not be null" );
-
-
- final UUID jobUUID = UUIDUtils.tryGetUUID( jobId );
-
- final StatusService.JobStatus
- job = getStatusService().getStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobUUID ).toBlocking().lastOrDefault(
- null );
-
- Preconditions.checkNotNull( job, "job with id '" + jobId + "' does not exist" );
-
-
- return createResult( job, callback );
- }
-
-
- private ConnectionService getConnectionService() {
- return injector.getInstance( ConnectionServiceImpl.class );
- }
-
-
- private StatusService getStatusService() {
- return injector.getInstance( StatusService.class );
- }
-
-
-
- /**
- * Execute the request and return the response.
- */
- private JSONWithPadding executeAndCreateResponse( final UUID applicationId, final String callback ) {
-
- final Observable<ApplicationScope> applicationScopeObservable =
- Observable.just( CpNamingUtils.getApplicationScope( applicationId ) );
-
- final UUID jobId = UUIDGenerator.newTimeUUID();
-
- final StatusService statusService = getStatusService();
- final ConnectionService connectionService = getConnectionService();
-
- final AtomicLong count = new AtomicLong( 0 );
-
- //start de duping and run in the background
- connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
- .doOnNext( buffer -> {
-
-
- final long runningTotal = count.addAndGet( buffer.size() );
-
- final Map<String, Object> status = new HashMap<String, Object>() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", System.currentTimeMillis() );
- }};
-
- statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
- StatusService.Status.INPROGRESS, status ).toBlocking().lastOrDefault( null );
- } ).doOnSubscribe( () -> {
- statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.STARTED,
- new HashMap<>() ).toBlocking().lastOrDefault( null );
- } ).doOnCompleted( () -> {
-
- final long runningTotal = count.get();
-
- final Map<String, Object> status = new HashMap<String, Object>() {{
- put( "countProcessed", runningTotal );
- put( "updatedTimestamp", System.currentTimeMillis() );
- }};
-
- statusService
- .setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.COMPLETE, status );
- } ).subscribeOn( Schedulers.newThread() ).subscribe();
-
-
- final StatusService.JobStatus status = new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
- return createResult( status, callback );
- }
-
-
- /**
- * Create a response with the specified data.
- * @param jobStatus
- * @param callback
- * @return
- */
- private JSONWithPadding createResult(final StatusService.JobStatus jobStatus, final String callback){
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "de-dup connections" );
- response.setProperty( "status", jobStatus );
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-}
-
-
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/DatabaseResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/DatabaseResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/DatabaseResource.java
deleted file mode 100644
index a8c5fee..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/DatabaseResource.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.rest;
-
-
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-
-import com.sun.jersey.api.json.JSONWithPadding;
-
-
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class DatabaseResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( DatabaseResource.class );
-
-
- public DatabaseResource() {
- logger.info( "DatabaseResource initialized" );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "setup" )
- public JSONWithPadding runDatabaseSetup( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "cassandra setup" );
-
- logger.info( "Setting up Cassandra" );
-
-
- emf.setup();
-
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "bootstrap" )
- public JSONWithPadding runSystemSetup( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "cassandra setup" );
-
- logger.info( "Setting up Cassandra" );
-
-
- emf.boostrap();
- management.setup();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-}
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
deleted file mode 100644
index a4a56bc..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.rest;
-
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
-import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
-import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl;
-import org.apache.usergrid.corepersistence.index.ReIndexService;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.utils.ConversionUtils;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-
-/**
- * system/index/otherstuff
- */
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class IndexResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( IndexResource.class );
- private static final String UPDATED_FIELD = "updated";
-
-
-
- public IndexResource() {
- super();
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "rebuild" )
- public JSONWithPadding rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
-
- logger.info("Rebuilding all applications");
-
- final ReIndexRequestBuilder request = createRequest();
-
- return executeAndCreateResponse( request, callback );
- }
-
- @RequireSystemAccess
- @GET
- @Path( "rebuild/{jobId}" )
- public JSONWithPadding rebuildIndexesGet(@PathParam( "jobId" ) String jobId, @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
- logger.info("Getting status for index jobs");
-
- Preconditions
- .checkNotNull(jobId, "query param jobId must not be null" );
-
-
- ReIndexService.ReIndexStatus status = getReIndexService().getStatus(jobId);
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "rebuild indexes" );
- response.setProperty( "jobId", status.getJobId() );
- response.setProperty( "status", status.getStatus() );
- response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
- response.setProperty( "numberQueued", status.getNumberProcessed() );
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
- @RequireSystemAccess
- @PUT
- @Path( "rebuild" )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
-
- logger.info( "Resuming rebuilding all applications" );
- final ReIndexRequestBuilder request = createRequest();
-
- return executeResumeAndCreateResponse( payload, request, callback );
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
-
- throws Exception {
-
-
- logger.info( "Rebuilding application {}", applicationIdStr );
-
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
-
- return executeAndCreateResponse( request, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
- @PathParam( "applicationId" ) String applicationIdStr,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
- @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
-
- throws Exception {
-
- logger.info( "Resuming rebuilding application {}", applicationIdStr );
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
-
- return executeResumeAndCreateResponse( payload, request, callback );
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
-
- logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- final ReIndexRequestBuilder request =
- createRequest().withApplicationId( appId ).withCollection( collectionName );
-
- return executeAndCreateResponse( request, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
- public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
- @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- logger.info( "Resuming rebuilding collection {} in application {}", collectionName, applicationIdStr );
-
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- final ReIndexRequestBuilder request =
- createRequest().withApplicationId( appId ).withCollection( collectionName );
-
- return executeResumeAndCreateResponse( payload, request, callback );
- }
-
-
- @RequireSystemAccess
- @POST
- @Path( "rebuild/management" )
- public JSONWithPadding rebuildInternalIndexesPost(
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception {
-
-
- final UUID managementAppId = emf.getManagementAppId();
-
- logger.info( "Rebuilding management application with id {} ", managementAppId );
- final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
-
- return executeAndCreateResponse( request, callback );
- }
-
-
- @RequireSystemAccess
- @PUT
- @Path( "rebuild/management" )
- public JSONWithPadding rebuildInternalIndexesPut( final Map<String, Object> payload,
- @QueryParam( "callback" ) @DefaultValue( "callback" )
- String callback ) throws Exception {
-
-
- final UUID managementAppId = emf.getManagementAppId();
-
- logger.info( "Resuming rebuilding management application with id {} ", managementAppId );
- final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
-
- return executeResumeAndCreateResponse( payload, request, callback );
- }
-
-
- @RequireSystemAccess
- @POST
- @Path(RootResource.APPLICATION_ID_PATH)
- public JSONWithPadding addIndex( @Context UriInfo ui,
- @PathParam( "applicationId" ) final String applicationIdStr,
- Map<String, Object> config,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- Preconditions
- .checkNotNull( config, "Payload for config is null, please pass {replicas:int, shards:int} in body" );
-
- ApiResponse response = createApiResponse();
-
- if ( !config.containsKey( "replicas" ) || !config.containsKey( "shards" ) ||
- !( config.get( "replicas" ) instanceof Integer ) || !( config.get( "shards" ) instanceof Integer ) ) {
- throw new IllegalArgumentException( "body must contains 'replicas' of type int and 'shards' of type int" );
- }
-
- if ( !config.containsKey( "indexSuffix" ) ) {
- throw new IllegalArgumentException( "Please add an indexSuffix to your post" );
- }
- final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
-
- if(appId == null){
- throw new IllegalArgumentException("app id was not parsed");
- }
-
- EntityManager em = emf.getEntityManager(appId);
- em.addIndex(config.get("indexSuffix").toString(), (int) config.get("shards"),
- (int) config.get("replicas"), (String) config.get("writeConsistency"));
- response.setAction( "Add index to alias" );
-
- return new JSONWithPadding( response, callback );
- }
-
-
- private ReIndexService getReIndexService() {
- return injector.getInstance( ReIndexService.class );
- }
-
-
- private ReIndexRequestBuilder createRequest() {
- //TODO: wire this up through spring, and in the future guice.
- return new ReIndexRequestBuilderImpl();
- }
-
-
- private JSONWithPadding executeResumeAndCreateResponse( final Map<String, Object> payload,
- final ReIndexRequestBuilder request,
- final String callback ) {
-
- Map<String,Object> newPayload = payload;
- if(newPayload == null || !payload.containsKey( UPDATED_FIELD )){
- newPayload = new HashMap<>(1);
- newPayload.put(UPDATED_FIELD,0);
- }
-
- Preconditions.checkArgument(newPayload.get(UPDATED_FIELD) instanceof Number,
- "You must specified the field \"updated\" in the payload and it must be a timestamp" );
-
- //add our updated timestamp to the request
- if ( newPayload.containsKey( UPDATED_FIELD ) ) {
- final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_FIELD));
- request.withStartTimestamp( timestamp );
- }
-
- return executeAndCreateResponse( request, callback );
- }
-
-
- /**
- * Execute the request and return the response.
- */
- private JSONWithPadding executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
-
-
- final ReIndexService.ReIndexStatus status = getReIndexService().rebuildIndex( request );
-
- final ApiResponse response = createApiResponse();
-
- response.setAction( "rebuild indexes" );
- response.setProperty( "jobId", status.getJobId() );
- response.setProperty( "status", status.getStatus() );
- response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
- response.setProperty( "numberQueued", status.getNumberProcessed() );
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
deleted file mode 100644
index 7410927..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/MigrateResource.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.rest;
-
-
-import java.util.Map;
-import java.util.Set;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-
-import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.base.Preconditions;
-import com.google.inject.Injector;
-import com.sun.jersey.api.json.JSONWithPadding;
-
-
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON,
- "application/javascript",
- "application/x-javascript",
- "text/ecmascript",
- "application/ecmascript",
- "text/jscript"
-} )
-public class MigrateResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( MigrateResource.class );
-
- public MigrateResource() {
- logger.info( "SystemResource initialized" );
- }
-
- @Autowired
- private Injector guiceInjector;
-
- @RequireSystemAccess
- @PUT
- @Path( "run" )
- public JSONWithPadding migrateData( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "Migrate Data" );
- //TODO make this use the task scheduler
-
-
- final Thread migrate = new Thread() {
-
- @Override
- public void run() {
-
- logger.info( "Migrating Schema" );
-
- try {
- getMigrationManager().migrate();
- }
- catch ( Exception e ) {
- logger.error( "Unable to migrate data", e );
- }
-
- logger.info( "Migrating Data" );
-
- try {
- getDataMigrationManager().migrate();
- }
- catch ( Exception e ) {
- logger.error( "Unable to migrate data", e );
- }
- }
- };
-
- migrate.setName( "Index migrate data formats" );
- migrate.setDaemon( true );
- migrate.start();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
- @RequireSystemAccess
- @PUT
- @Path( "run/{pluginName}" )
- public JSONWithPadding migrateData(@PathParam("pluginName") String pluginName , @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- if(!getDataMigrationManager().pluginExists(pluginName)){
- throw new IllegalArgumentException("Plugin doesn't exits name:"+pluginName);
- }
-
- ApiResponse response = createApiResponse();
- response.setAction( "Migrate Data: "+ pluginName );
- //TODO make this use the task scheduler
-
-
-
-
- final Thread migrate = new Thread() {
-
- @Override
- public void run() {
-
- logger.info( "Migrating Data for plugin: " + pluginName );
-
- try {
- getDataMigrationManager().migrate(pluginName);
- }
- catch ( Exception e ) {
- logger.error( "Unable to migrate data for plugin: " + pluginName, e );
- }
- }
- };
-
- migrate.setName( "Index migrate data formats: "+pluginName );
- migrate.setDaemon( true );
- migrate.start();
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
- @RequireSystemAccess
- @PUT
- @Path( "set" )
- public JSONWithPadding setMigrationVersion(
- @Context UriInfo ui, Map<String, Object> json,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback )
- throws Exception {
-
- logger.debug( "setMigrationVersion" );
-
- Preconditions.checkNotNull( json, "You must provide a json body" );
- Preconditions.checkArgument( json.keySet().size() > 0, "You must specify at least one module and version" );
-
- ApiResponse response = createApiResponse();
- response.setAction("Set Migration Versions");
-
- ObjectNode node = JsonNodeFactory.instance.objectNode();
-
- final DataMigrationManager dataMigrationManager = getDataMigrationManager();
- final Set<String> plugins = dataMigrationManager.getPluginNames();
-
- /**
- * Set the migration version for the plugins specified
- */
- for ( final String key : json.keySet() ) {
-
- int version = ( int ) json.get( key );
-
- dataMigrationManager.resetToVersion(key, version);
- }
-
-
- /**
- * Echo back a response of the current versions for all plugins
- */
- for(final String pluginName: plugins){
- node.put(pluginName, dataMigrationManager.getCurrentVersion(pluginName));
- }
-
-
- response.setData( node );
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @GET
- @Path( "status" )
- public JSONWithPadding migrateStatus(
- @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "Migrate Schema indexes" );
-
- ObjectNode node = JsonNodeFactory.instance.objectNode();
-
-
-
- final DataMigrationManager dataMigrationManager = getDataMigrationManager();
-
- final Set<String> plugins = dataMigrationManager.getPluginNames();
-
- for(final String pluginName: plugins){
- node.put( pluginName, dataMigrationManager.getCurrentVersion( pluginName ) );
- }
-
- response.setData( node );
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireSystemAccess
- @GET
- @Path( "count" )
- public JSONWithPadding migrateCount(
- @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "Current entity count in system" );
-
- response.setProperty( "count", emf.performEntityCount() );
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
- /**
- * Get the schema migration manager
- */
- private MigrationManager getMigrationManager() {
- return guiceInjector.getInstance( MigrationManager.class );
- }
-
- /**
- * Get the Data migration manager
- */
- private DataMigrationManager getDataMigrationManager() {
- return guiceInjector.getInstance( DataMigrationManager.class );
- }
-}
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
deleted file mode 100644
index aaee596..0000000
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/SystemResource.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.rest;
-
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-
-import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-
-import com.sun.jersey.api.json.JSONWithPadding;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-
-@Path( "/system" )
-@Component
-@Scope( "singleton" )
-@Produces( {
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-} )
-public class SystemResource extends AbstractContextResource {
-
- private static final Logger logger = LoggerFactory.getLogger( SystemResource.class );
-
-
- public SystemResource() {
- logger.info( "SystemResource initialized" );
- }
-
-
- @RequireSystemAccess
- @GET
- @Path( "superuser/setup" )
- public JSONWithPadding getSetupSuperuser( @Context UriInfo ui,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "superuser setup" );
-
- logger.info( "Setting up Superuser" );
-
- try {
- management.provisionSuperuser();
- }
- catch ( Exception e ) {
- logger.error( "Unable to complete superuser setup", e );
- }
-
- response.setSuccess();
-
- return new JSONWithPadding( response, callback );
- }
-
-
-
- @Path( "migrate" )
- public MigrateResource migrate() {
- return getSubResource( MigrateResource.class );
- }
-
-
- @Path( "index" )
- public IndexResource index() { return getSubResource( IndexResource.class ); }
-
-
- @Path( "database" )
- public DatabaseResource database() {
- return getSubResource( DatabaseResource.class );
- }
-
- @Path( "applications" )
- public ApplicationsResource applications() {
- return getSubResource( ApplicationsResource.class );
- }
-
-
- @Path( "connection" )
- public ConnectionResource connection() { return getSubResource( ConnectionResource.class ); }
-
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
new file mode 100644
index 0000000..82088f9
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
@@ -0,0 +1,178 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.rest.system;
+
+import com.sun.jersey.api.json.JSONWithPadding;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Classy class class.
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class ApplicationsResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(ApplicationsResource.class);
+
+
+ public ApplicationsResource() {
+
+ logger.info( "ApplicationsResource initialized" );
+ } {
+
+ }
+
+ @RequireSystemAccess
+ @DELETE
+ @Path( "{applicationId}" )
+ public JSONWithPadding clearApplication( @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @QueryParam( "confirmApplicationName" ) String confirmApplicationName,
+ @QueryParam( "limit" ) int limit,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ if(confirmApplicationName == null){
+ throw new IllegalArgumentException("please make add a QueryString for confirmApplicationName");
+ }
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final EntityManager em = emf.getEntityManager(applicationId);
+ final String name = em.getApplication().getApplicationName();
+ if(!name.toLowerCase().equals(confirmApplicationName.toLowerCase())){
+ throw new IllegalArgumentException("confirmApplicationName: " + confirmApplicationName + " does not equal " + name);
+ }
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ logger.info("clearing up application");
+
+
+ final Thread delete = new Thread() {
+
+ @Override
+ public void run() {
+ final AtomicInteger itemsDeleted = new AtomicInteger(0);
+ try {
+ management.deleteAllEntities(applicationId, limit)
+ .map(id -> itemsDeleted.incrementAndGet())
+ .doOnNext(count -> {
+ if( count % 100 == 0 ){
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("count",itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
+ statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map)
+ .subscribe();//do not want to throw this exception
+ }
+ })
+ .doOnCompleted(() -> {
+ Map<String, Object> map = new LinkedHashMap<>();
+ map.put("count", itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
+ statusService.setStatus(applicationId, jobId, StatusService.Status.COMPLETE, map)
+ .toBlocking().lastOrDefault(null);//want to rethrow this exception
+ })
+ .toBlocking().lastOrDefault(null);//expecting exception to be caught if job fails
+
+ } catch ( Exception e ) {
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("exception",e);
+ try {
+ statusService.setStatus(applicationId, jobId, StatusService.Status.FAILED, map).toBlocking().lastOrDefault(null);//leave as subscribe if fails retry
+ }catch (Exception subE){
+ logger.error("failed to update status "+jobId,subE);
+ }
+ logger.error( "Failed to delete appid:"+applicationId + " jobid:"+jobId+" count:"+itemsDeleted, e );
+ }
+ }
+ };
+
+ delete.setName("Delete for app : " + applicationId + " job: " + jobId);
+ delete.setDaemon(true);
+ delete.start();
+
+ try {
+ //should throw exception if can't start
+ statusService.setStatus(applicationId, jobId, StatusService.Status.STARTED, new LinkedHashMap<>()).toBlocking().lastOrDefault(null);
+ }catch (Exception e){
+ logger.error("failed to set status for " + jobId, e);
+ }
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put("status",StatusService.Status.STARTED);
+ response.setData(data);
+ response.setSuccess();
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "{applicationId}/job/{jobId}" )
+ public JSONWithPadding getStatus( @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @PathParam("jobId") UUID jobId,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception{
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ StatusService.JobStatus jobStatus = statusService.getStatus(applicationId, jobId).toBlocking().lastOrDefault(null);
+
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put( "status", jobStatus.getStatus().toString() );
+ data.put( "metadata", jobStatus.getData() );
+ response.setData(data);
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
new file mode 100644
index 0000000..6e683ed
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ConnectionResource.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.rest.system;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.query.Identifier;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.RootResource;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.json.JSONWithPadding;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+
+/**
+ * system/index/otherstuff
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class ConnectionResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( ConnectionResource.class );
+
+ public ConnectionResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "dedup/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding all applications" );
+
+ final UUID applicationId = UUIDUtils.tryGetUUID( applicationIdStr );
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+
+ return executeAndCreateResponse( applicationId, callback );
+ }
+
+
+ @RequireSystemAccess
+ @GET
+ @Path( "dedup/{jobId: " + Identifier.UUID_REX + "}" )
+ public JSONWithPadding rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+ logger.info( "Getting status for index jobs" );
+
+ Preconditions.checkNotNull( jobId, "query param jobId must not be null" );
+
+
+ final UUID jobUUID = UUIDUtils.tryGetUUID( jobId );
+
+ final StatusService.JobStatus
+ job = getStatusService().getStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobUUID ).toBlocking().lastOrDefault(
+ null );
+
+ Preconditions.checkNotNull( job, "job with id '" + jobId + "' does not exist" );
+
+
+ return createResult( job, callback );
+ }
+
+
+ private ConnectionService getConnectionService() {
+ return injector.getInstance( ConnectionServiceImpl.class );
+ }
+
+
+ private StatusService getStatusService() {
+ return injector.getInstance( StatusService.class );
+ }
+
+
+
+ /**
+ * Execute the request and return the response.
+ */
+ private JSONWithPadding executeAndCreateResponse( final UUID applicationId, final String callback ) {
+
+ final Observable<ApplicationScope> applicationScopeObservable =
+ Observable.just( CpNamingUtils.getApplicationScope( applicationId ) );
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final StatusService statusService = getStatusService();
+ final ConnectionService connectionService = getConnectionService();
+
+ final AtomicLong count = new AtomicLong( 0 );
+
+ //start de duping and run in the background
+ connectionService.deDupeConnections( applicationScopeObservable ).buffer( 10, TimeUnit.SECONDS, 1000 )
+ .doOnNext( buffer -> {
+
+
+ final long runningTotal = count.addAndGet( buffer.size() );
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put( "countProcessed", runningTotal );
+ put( "updatedTimestamp", System.currentTimeMillis() );
+ }};
+
+ statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
+ StatusService.Status.INPROGRESS, status ).toBlocking().lastOrDefault( null );
+ } ).doOnSubscribe( () -> {
+ statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.STARTED,
+ new HashMap<>() ).toBlocking().lastOrDefault( null );
+ } ).doOnCompleted( () -> {
+
+ final long runningTotal = count.get();
+
+ final Map<String, Object> status = new HashMap<String, Object>() {{
+ put( "countProcessed", runningTotal );
+ put( "updatedTimestamp", System.currentTimeMillis() );
+ }};
+
+ statusService
+ .setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.COMPLETE, status );
+ } ).subscribeOn( Schedulers.newThread() ).subscribe();
+
+
+ final StatusService.JobStatus status = new StatusService.JobStatus( jobId, StatusService.Status.STARTED, new HashMap<>( ) );
+ return createResult( status, callback );
+ }
+
+
+ /**
+ * Create a response with the specified data.
+ * @param jobStatus
+ * @param callback
+ * @return
+ */
+ private JSONWithPadding createResult(final StatusService.JobStatus jobStatus, final String callback){
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "de-dup connections" );
+ response.setProperty( "status", jobStatus );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
new file mode 100644
index 0000000..42a63ca
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/DatabaseResource.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.system;
+
+
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+
+import com.sun.jersey.api.json.JSONWithPadding;
+
+
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class DatabaseResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( DatabaseResource.class );
+
+
+ public DatabaseResource() {
+ logger.info( "DatabaseResource initialized" );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "setup" )
+ public JSONWithPadding runDatabaseSetup( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "cassandra setup" );
+
+ logger.info( "Setting up Cassandra" );
+
+
+ emf.setup();
+
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "bootstrap" )
+ public JSONWithPadding runSystemSetup( @Context UriInfo ui,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ ApiResponse response = createApiResponse();
+ response.setAction( "cassandra setup" );
+
+ logger.info( "Setting up Cassandra" );
+
+
+ emf.boostrap();
+ management.setup();
+
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7a4f36ab/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
new file mode 100644
index 0000000..8e2946c
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
@@ -0,0 +1,331 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.rest.system;
+
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.json.JSONWithPadding;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.index.utils.ConversionUtils;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.RootResource;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+
+/**
+ * system/index/otherstuff
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class IndexResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger( IndexResource.class );
+ private static final String UPDATED_FIELD = "updated";
+
+
+
+ public IndexResource() {
+ super();
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild" )
+ public JSONWithPadding rebuildIndexesPost( @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info("Rebuilding all applications");
+
+ final ReIndexRequestBuilder request = createRequest();
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "rebuild/{jobId}" )
+ public JSONWithPadding rebuildIndexesGet(@PathParam( "jobId" ) String jobId, @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+ logger.info("Getting status for index jobs");
+
+ Preconditions
+ .checkNotNull(jobId, "query param jobId must not be null" );
+
+
+ ReIndexService.ReIndexStatus status = getReIndexService().getStatus(jobId);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild" )
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Resuming rebuilding all applications" );
+ final ReIndexRequestBuilder request = createRequest();
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexesPut( @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+
+ logger.info( "Rebuilding application {}", applicationIdStr );
+
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH )
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) String applicationIdStr,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
+ @QueryParam( "delay" ) @DefaultValue( "10" ) final long delay )
+
+ throws Exception {
+
+ logger.info( "Resuming rebuilding application {}", applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( appId );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
+ public JSONWithPadding rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+
+ logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
+ public JSONWithPadding rebuildIndexesPut( final Map<String, Object> payload,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ logger.info( "Resuming rebuilding collection {} in application {}", collectionName, applicationIdStr );
+
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( appId ).withCollection( collectionName );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path( "rebuild/management" )
+ public JSONWithPadding rebuildInternalIndexesPost(
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception {
+
+
+ final UUID managementAppId = emf.getManagementAppId();
+
+ logger.info( "Rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @PUT
+ @Path( "rebuild/management" )
+ public JSONWithPadding rebuildInternalIndexesPut( final Map<String, Object> payload,
+ @QueryParam( "callback" ) @DefaultValue( "callback" )
+ String callback ) throws Exception {
+
+
+ final UUID managementAppId = emf.getManagementAppId();
+
+ logger.info( "Resuming rebuilding management application with id {} ", managementAppId );
+ final ReIndexRequestBuilder request = createRequest().withApplicationId( managementAppId );
+
+ return executeResumeAndCreateResponse( payload, request, callback );
+ }
+
+
+ @RequireSystemAccess
+ @POST
+ @Path(RootResource.APPLICATION_ID_PATH)
+ public JSONWithPadding addIndex( @Context UriInfo ui,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ Map<String, Object> config,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ throws Exception {
+
+ Preconditions
+ .checkNotNull( config, "Payload for config is null, please pass {replicas:int, shards:int} in body" );
+
+ ApiResponse response = createApiResponse();
+
+ if ( !config.containsKey( "replicas" ) || !config.containsKey( "shards" ) ||
+ !( config.get( "replicas" ) instanceof Integer ) || !( config.get( "shards" ) instanceof Integer ) ) {
+ throw new IllegalArgumentException( "body must contains 'replicas' of type int and 'shards' of type int" );
+ }
+
+ if ( !config.containsKey( "indexSuffix" ) ) {
+ throw new IllegalArgumentException( "Please add an indexSuffix to your post" );
+ }
+ final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
+ if(appId == null){
+ throw new IllegalArgumentException("app id was not parsed");
+ }
+
+ EntityManager em = emf.getEntityManager(appId);
+ em.addIndex(config.get("indexSuffix").toString(), (int) config.get("shards"),
+ (int) config.get("replicas"), (String) config.get("writeConsistency"));
+ response.setAction( "Add index to alias" );
+
+ return new JSONWithPadding( response, callback );
+ }
+
+
+ private ReIndexService getReIndexService() {
+ return injector.getInstance( ReIndexService.class );
+ }
+
+
+ private ReIndexRequestBuilder createRequest() {
+ //TODO: wire this up through spring, and in the future guice.
+ return new ReIndexRequestBuilderImpl();
+ }
+
+
+ private JSONWithPadding executeResumeAndCreateResponse( final Map<String, Object> payload,
+ final ReIndexRequestBuilder request,
+ final String callback ) {
+
+ Map<String,Object> newPayload = payload;
+ if(newPayload == null || !payload.containsKey( UPDATED_FIELD )){
+ newPayload = new HashMap<>(1);
+ newPayload.put(UPDATED_FIELD,0);
+ }
+
+ Preconditions.checkArgument(newPayload.get(UPDATED_FIELD) instanceof Number,
+ "You must specified the field \"updated\" in the payload and it must be a timestamp" );
+
+ //add our updated timestamp to the request
+ if ( newPayload.containsKey( UPDATED_FIELD ) ) {
+ final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_FIELD));
+ request.withStartTimestamp( timestamp );
+ }
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+
+ /**
+ * Execute the request and return the response.
+ */
+ private JSONWithPadding executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
+
+
+ final ReIndexService.ReIndexStatus status = getReIndexService().rebuildIndex( request );
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return new JSONWithPadding( response, callback );
+ }
+}
[22/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/413/head' of github.com:apache/usergrid into 2.1-release
Posted by to...@apache.org.
Merge commit 'refs/pull/413/head' of github.com:apache/usergrid into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2aec8a0f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2aec8a0f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2aec8a0f
Branch: refs/heads/master
Commit: 2aec8a0fa3e607c67dafd5a8e041674ac9859237
Parents: 07d2ad3 752bf4e
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Oct 26 18:39:27 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Oct 26 18:39:27 2015 -0600
----------------------------------------------------------------------
.../organizations/OrganizationsResource.java | 42 +++++++++++++-------
1 file changed, 28 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[26/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/414/head' of github.com:apache/usergrid into 2.1-release
Posted by to...@apache.org.
Merge commit 'refs/pull/414/head' of github.com:apache/usergrid into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e018c1e0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e018c1e0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e018c1e0
Branch: refs/heads/master
Commit: e018c1e0d65e3af339e36c69a4e59fe2d2a83167
Parents: 2aec8a0 7a4f36a
Author: Michael Russo <mi...@gmail.com>
Authored: Tue Oct 27 10:54:04 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Tue Oct 27 10:54:04 2015 -0700
----------------------------------------------------------------------
.../service/StatusServiceImpl.java | 3 +-
.../usergrid/rest/ApplicationsResource.java | 178 ----------
.../apache/usergrid/rest/DatabaseResource.java | 98 ------
.../org/apache/usergrid/rest/IndexResource.java | 328 ------------------
.../apache/usergrid/rest/MigrateResource.java | 268 ---------------
.../apache/usergrid/rest/SystemResource.java | 103 ------
.../rest/system/ApplicationsResource.java | 178 ++++++++++
.../rest/system/ConnectionResource.java | 202 +++++++++++
.../usergrid/rest/system/DatabaseResource.java | 100 ++++++
.../usergrid/rest/system/IndexResource.java | 331 +++++++++++++++++++
.../usergrid/rest/system/MigrateResource.java | 270 +++++++++++++++
.../usergrid/rest/system/SystemResource.java | 108 ++++++
12 files changed, 1191 insertions(+), 976 deletions(-)
----------------------------------------------------------------------
[43/50] [abbrv] usergrid git commit: Add dup edges so we exercise the
dedup logic.
Posted by to...@apache.org.
Add dup edges so we exercise the dedup logic.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9ada6ea3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9ada6ea3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9ada6ea3
Branch: refs/heads/master
Commit: 9ada6ea3732a3bfb72cc6a48d12353d0ac832ce6
Parents: b816a18
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Oct 30 08:30:18 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Oct 30 08:30:18 2015 -0400
----------------------------------------------------------------------
stack/scripts/create_test_data.py | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9ada6ea3/stack/scripts/create_test_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/create_test_data.py b/stack/scripts/create_test_data.py
index ef07254..3a913ca 100644
--- a/stack/scripts/create_test_data.py
+++ b/stack/scripts/create_test_data.py
@@ -155,6 +155,8 @@ class Creator:
targetType = appName + "_col1"
url = appUrl + "/" + sourceCollection + "/" + sourceName + "/has/" + targetType + "/" + targetName
r = requests.post(url=url + "?access_token=" + accessToken)
+ r = requests.post(url=url + "?access_token=" + accessToken)
+ r = requests.post(url=url + "?access_token=" + accessToken)
if ( r.status_code >= 400 ):
print "Error connecting entity " + sourceName + " to " + targetName + ": " + r.text
print "url is: " + url
@@ -168,6 +170,8 @@ class Creator:
targetType = appName + "_col2"
url = appUrl + "/" + sourceCollection + "/" + sourceName + "/has/" + targetType + "/" + targetName
r = requests.post(url=url + "?access_token=" + accessToken)
+ r = requests.post(url=url + "?access_token=" + accessToken)
+ r = requests.post(url=url + "?access_token=" + accessToken)
if ( r.status_code >= 400 ):
print "Error connecting entity " + sourceName + " to " + targetName + ": " + r.text
print "url is: " + url
[29/50] [abbrv] usergrid git commit: Merge branch
'delete-event-updates' into 2.1-release
Posted by to...@apache.org.
Merge branch 'delete-event-updates' into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/26860545
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/26860545
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/26860545
Branch: refs/heads/master
Commit: 26860545ece19361569370bc63496a01bb48738d
Parents: e018c1e 76476f1
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Oct 27 14:40:23 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Oct 27 14:40:23 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 68 ++++++++++--------
.../asyncevents/AsyncEventService.java | 5 ++
.../asyncevents/InMemoryAsyncEventService.java | 5 ++
.../index/IndexProcessorFig.java | 13 +++-
.../read/traverse/AbstractReadGraphFilter.java | 72 +++++++++++++++++---
.../traverse/ReadGraphCollectionFilter.java | 10 ++-
.../traverse/ReadGraphConnectionFilter.java | 10 ++-
.../impl/stage/NodeDeleteListenerImpl.java | 27 +++-----
8 files changed, 150 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
[46/50] [abbrv] usergrid git commit: Add call to database bootstrap
Posted by to...@apache.org.
Add call to database bootstrap
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9b589d37
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9b589d37
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9b589d37
Branch: refs/heads/master
Commit: 9b589d37bba835cdc8fc8912919f7929fa1bfb21
Parents: 8737f69
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Oct 30 13:49:25 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Oct 30 13:49:25 2015 -0400
----------------------------------------------------------------------
stack/scripts/multitenant_migrate.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9b589d37/stack/scripts/multitenant_migrate.py
----------------------------------------------------------------------
diff --git a/stack/scripts/multitenant_migrate.py b/stack/scripts/multitenant_migrate.py
index 6955de7..5f1918e 100644
--- a/stack/scripts/multitenant_migrate.py
+++ b/stack/scripts/multitenant_migrate.py
@@ -226,6 +226,7 @@ class Migrate:
# Init the migration system as this is the first migration done on the cluster
self.run_database_setup()
+ self.run_database_bootstrap()
migration_system_updated = self.is_migration_system_updated()
[47/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/418/head' of github.com:apache/usergrid into 2.1-release
Posted by to...@apache.org.
Merge commit 'refs/pull/418/head' of github.com:apache/usergrid into 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/40dfa064
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/40dfa064
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/40dfa064
Branch: refs/heads/master
Commit: 40dfa0647709881170ea0874036d0ea84875148f
Parents: 3d12057 9b589d3
Author: Michael Russo <mi...@gmail.com>
Authored: Fri Oct 30 12:33:48 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Fri Oct 30 12:33:48 2015 -0700
----------------------------------------------------------------------
.../rest/system/ConnectionResource.java | 48 +-
stack/scripts/create_test_data.py | 222 ++++++
stack/scripts/migrate_entity_data.py | 18 +-
stack/scripts/multitenant_migrate.py | 702 +++++++++++++++++++
4 files changed, 965 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
[45/50] [abbrv] usergrid git commit: Addresses comments in review
Posted by to...@apache.org.
Addresses comments in review
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3d120578
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3d120578
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3d120578
Branch: refs/heads/master
Commit: 3d1205784528b5026695910943243a7a38fcbdde
Parents: ba629da
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 30 10:51:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 30 10:51:58 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/rest/applications/users/UserResource.java | 6 +++---
.../rest/applications/collection/users/UserResourceIT.java | 2 --
2 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3d120578/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index 1116469..fb2962e 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -173,11 +173,11 @@ public class UserResource extends ServiceResource {
public JSONWithPadding getUserCredentials(@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
- logger.info( "UserResource.setUserPassword" );
+ logger.info( "UserResource.getUserCredentials" );
final ApiResponse response = createApiResponse();
- response.setAction( "get user password" );
+ response.setAction( "get user credentials" );
final UUID applicationId = getApplicationId();
final UUID targetUserId = getUserUuid();
@@ -210,7 +210,7 @@ public class UserResource extends ServiceResource {
@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
- logger.info( "UserResource.setUserPassword" );
+ logger.info( "UserResource.setUserCredentials" );
if ( json == null ) {
return null;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3d120578/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index deda5eb..f258f94 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@ -1154,7 +1154,6 @@ public class UserResourceIT extends AbstractRestIT {
//here we modify the hash a little, this way we can break password validation, then re-set it to ensure we're actually updating the credentials info correctly.
final String borkedSecret = originalSecret.substring( 0, originalSecret.length() -1 );
- credentials.put( "credentials", borkedSecret );
credentials.put( "secret", borkedSecret );
//now PUT it
@@ -1192,7 +1191,6 @@ public class UserResourceIT extends AbstractRestIT {
//now put the correct one
- credentials.put( "credentials", originalSecret );
credentials.put( "secret", originalSecret );
[42/50] [abbrv] usergrid git commit: Merge branch '2.1-release' into
multitenant-migration
Posted by to...@apache.org.
Merge branch '2.1-release' into multitenant-migration
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b816a188
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b816a188
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b816a188
Branch: refs/heads/master
Commit: b816a188bade82ec1044e4dfb80220da67182b3c
Parents: 8af152e c652171
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 20:24:14 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 20:24:14 2015 -0400
----------------------------------------------------------------------
.../corepersistence/index/IndexServiceImpl.java | 2 +-
.../persistence/index/CandidateResult.java | 11 +-
.../persistence/index/EntityIndexBatch.java | 2 +-
.../usergrid/persistence/index/IndexFig.java | 6 +
.../index/impl/DeIndexOperation.java | 5 +
.../index/impl/EsEntityIndexBatchImpl.java | 34 ++++-
.../index/impl/EsEntityIndexImpl.java | 149 +++++++++++--------
.../persistence/index/impl/IndexingUtils.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 14 +-
9 files changed, 143 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
[21/50] [abbrv] usergrid git commit: Fixes bug in resources and tests
job status retrieval successfully
Posted by to...@apache.org.
Fixes bug in resources and tests job status retrieval successfully
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3b436243
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3b436243
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3b436243
Branch: refs/heads/master
Commit: 3b436243eb19ea03151aedf1ea13a63e2048f2f4
Parents: e41b5f0
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Oct 26 18:24:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Oct 26 18:24:58 2015 -0600
----------------------------------------------------------------------
.../java/org/apache/usergrid/rest/ConnectionResource.java | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b436243/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
index b6a38e1..dfa4781 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/ConnectionResource.java
@@ -43,6 +43,7 @@ import org.apache.usergrid.corepersistence.service.ConnectionServiceImpl;
import org.apache.usergrid.corepersistence.service.StatusService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
@@ -92,7 +93,7 @@ public class ConnectionResource extends AbstractContextResource {
@RequireSystemAccess
@GET
- @Path( "dedup/{jobId}" )
+ @Path( "dedup/{jobId: " + Identifier.UUID_REX + "}" )
public JSONWithPadding rebuildIndexesGet( @PathParam( "jobId" ) String jobId,
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
@@ -153,10 +154,10 @@ public class ConnectionResource extends AbstractContextResource {
}};
statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId,
- StatusService.Status.INPROGRESS, status );
+ StatusService.Status.INPROGRESS, status ).toBlocking().lastOrDefault( null );
} ).doOnSubscribe( () -> {
statusService.setStatus( CpNamingUtils.MANAGEMENT_APPLICATION_ID, jobId, StatusService.Status.STARTED,
- new HashMap<>() );
+ new HashMap<>() ).toBlocking().lastOrDefault( null );
} ).doOnCompleted( () -> {
final long runningTotal = count.get();
[40/50] [abbrv] usergrid git commit: Corrections to some comments.
Posted by to...@apache.org.
Corrections to some comments.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/57a613b5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/57a613b5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/57a613b5
Branch: refs/heads/master
Commit: 57a613b58c4cd886c518eaa63e4e8179b3a19cb0
Parents: f8c703c
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 29 17:58:10 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 29 17:58:10 2015 -0400
----------------------------------------------------------------------
stack/scripts/create_test_data.py | 23 ++++++++++++++---------
1 file changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/57a613b5/stack/scripts/create_test_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/create_test_data.py b/stack/scripts/create_test_data.py
index de85da0..ef07254 100644
--- a/stack/scripts/create_test_data.py
+++ b/stack/scripts/create_test_data.py
@@ -27,11 +27,16 @@ import json
# Caller must provide a "slug" string which will be used as a prefix for all names
#
# For example, if the slug is mytest then:
-# Orgs will be named mytest_org0 and mytest_org1
-# Apps will be named mytest_org0_app0 and so on
+#
+# Orgs will be named mytest_org0 and mytest_org1
+# Org admins will be named mytest_org0_admin and mytest_org1_admin (both with password test)
+#
+# Apps will be named mytest_org0_app0, mytest_org0_app1 and so on
# Collections will be named mytest_org0_app0_col0 and so on
-# Entities will be named mytest_org0_app0_col0_entity and so on
-# Org admins will be named mytest_org0_admin and mytest_org1_admin (both with password test)
+# Entities will be named mytest_org0_app0_col0_entity and so on
+#
+# All entities in collection 0 will be connected to entities in collection 1.
+# All entities in collection 1 will be connected to entities in collection 2.
def parse_args():
parser = argparse.ArgumentParser(description="Usergrid Test Data Creation Tool")
@@ -42,7 +47,7 @@ def parse_args():
default="http://localhost:8080")
parser.add_argument("--user",
- help="System Admin Credentials used to authenticate with Usergrid <user:pass>",
+ help="Superuser credentials used to authenticate with Usergrid <user:pass>",
type=str,
required=True)
@@ -56,7 +61,7 @@ def parse_args():
arg_vars = vars(my_args)
creds = arg_vars["user"].split(":")
if len(creds) != 2:
- print("Credentials not properly specified. Must be '-u <user:pass>'. Exiting...")
+ print("Credentials not properly specified. Must be '--user <user:pass>'. Exiting...")
exit_on_error()
else:
arg_vars["user"] = creds[0]
@@ -100,7 +105,7 @@ class Creator:
if ( r.status_code != 200 ):
print "Error logging into organization " + orgName + ": " + r.text
return
-
+
accessToken = r.json()["access_token"]
for appIndex in range(2):
@@ -115,11 +120,11 @@ class Creator:
print " Created app: " + orgName + "/" + appName
appUrl = self.endpoint + "/" + orgName + "/" + appName
- time.sleep(2)
+ time.sleep(2)
for userIndex in range(2):
userName = appName + "_user" + str(userIndex)
- email = userName + "@example.com"
+ email = userName + "@example.com"
url = appUrl + "/users?access_token=" + accessToken
body = json.dumps({"name":userName, "username":userName, "email":email, "password":"test"})
[12/50] [abbrv] usergrid git commit: Upgrades RX to latest stable
Posted by to...@apache.org.
Upgrades RX to latest stable
Adds deleted markers for the nodes on edges
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4e51d383
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4e51d383
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4e51d383
Branch: refs/heads/master
Commit: 4e51d3839cecc9d55ea0bc4f65d5610d29b34fb9
Parents: f73ac4c
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 23 14:16:17 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 23 14:16:17 2015 -0600
----------------------------------------------------------------------
.../read/traverse/AbstractReadGraphFilter.java | 2 +-
.../usergrid/persistence/graph/MarkedEdge.java | 13 +++++
.../graph/impl/GraphManagerImpl.java | 54 ++++++++++----------
.../graph/impl/SimpleMarkedEdge.java | 33 ++++++++++--
stack/corepersistence/pom.xml | 2 +-
5 files changed, 71 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4e51d383/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 88c912a..d3e0345 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -85,7 +85,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
/**
* TODO, pass a message with pointers to our cursor values to be generated later
*/
- return graphManager.loadEdgesFromSource( search ).onBackpressureBlock()
+ return graphManager.loadEdgesFromSource( search )
//set the edge state for cursors
.doOnNext( edge -> {
logger.trace( "Seeking over edge {}", edge );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4e51d383/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
index 4b5eeaa..da6fedb 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/MarkedEdge.java
@@ -38,4 +38,17 @@ public interface MarkedEdge extends Edge{
*/
boolean isDeleted();
+ /**
+ * Return true if the source node is deleted
+ * @return
+ */
+ boolean isSourceNodeDelete();
+
+ /**
+ * Return true if the target node is deleted
+ * @return
+ */
+ boolean isTargetNodeDeleted();
+
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4e51d383/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index c1e9cea..1bcb398 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -291,7 +291,7 @@ public class GraphManagerImpl implements GraphManager {
return storageEdgeSerialization.getEdgeVersions( scope, searchByEdge );
}
} ).buffer( graphFig.getScanPageSize() )
- .compose( new EdgeBufferFilter( searchByEdge.getMaxTimestamp(), searchByEdge.filterMarked() ) );
+ .compose( new EdgeBufferFilter( searchByEdge.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesVersionsTimer );
}
@@ -306,7 +306,7 @@ public class GraphManagerImpl implements GraphManager {
return storageEdgeSerialization.getEdgesFromSource( scope, search );
}
} ).buffer( graphFig.getScanPageSize() )
- .compose( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+ .compose( new EdgeBufferFilter( search.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesFromSourceTimer );
}
@@ -321,7 +321,7 @@ public class GraphManagerImpl implements GraphManager {
return storageEdgeSerialization.getEdgesToTarget( scope, search );
}
} ).buffer( graphFig.getScanPageSize() )
- .compose( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+ .compose( new EdgeBufferFilter( search.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesToTargetTimer );
@@ -337,7 +337,7 @@ public class GraphManagerImpl implements GraphManager {
return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope, search );
}
} ).buffer( graphFig.getScanPageSize() )
- .compose( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+ .compose( new EdgeBufferFilter( search.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesFromSourceByTypeTimer );
}
@@ -352,7 +352,7 @@ public class GraphManagerImpl implements GraphManager {
return storageEdgeSerialization.getEdgesToTargetBySourceType( scope, search );
}
} ).buffer( graphFig.getScanPageSize() )
- .compose( new EdgeBufferFilter( search.getMaxTimestamp(), search.filterMarked() ) );
+ .compose( new EdgeBufferFilter( search.filterMarked() ) );
return ObservableTimer.time( edges, loadEdgesToTargetByTypeTimer );
}
@@ -420,12 +420,10 @@ public class GraphManagerImpl implements GraphManager {
Observable.Transformer<List<MarkedEdge>, MarkedEdge> {//implements Func1<List<MarkedEdge>,
// Observable<MarkedEdge>> {
- private final long maxVersion;
private final boolean filterMarked;
- private EdgeBufferFilter( final long maxVersion, final boolean filterMarked ) {
- this.maxVersion = maxVersion;
+ private EdgeBufferFilter( final boolean filterMarked ) {
this.filterMarked = filterMarked;
}
@@ -444,23 +442,16 @@ public class GraphManagerImpl implements GraphManager {
final Observable<MarkedEdge> markedEdgeObservable = Observable.from( markedEdges );
- /**
- * We aren't going to filter anything, return exactly what we're passed
- */
- if(!filterMarked){
- return markedEdgeObservable;
- }
-
//We need to filter, perform that filter
final Map<Id, Long> markedVersions = nodeSerialization.getMaxVersions( scope, markedEdges );
- return markedEdgeObservable.filter( edge -> {
- final long edgeTimestamp = edge.getTimestamp();
+ return markedEdgeObservable.map( edge -> {
- //our edge needs to not be deleted and have a version that's > max Version
- if ( edge.isDeleted() ) {
- return false;
- }
+ /**
+ * Make sure we mark source and target deleted nodes as such
+ */
+
+ final long edgeTimestamp = edge.getTimestamp();
final Long sourceTimestamp = markedVersions.get( edge.getSourceNode() );
@@ -468,22 +459,29 @@ public class GraphManagerImpl implements GraphManager {
//the source Id has been marked for deletion. It's version is <= to the marked version for
// deletion,
// so we need to discard it
- if ( sourceTimestamp != null && Long.compare( edgeTimestamp, sourceTimestamp ) < 1 ) {
- return false;
- }
+ final boolean isSourceDeleted = ( sourceTimestamp != null && Long.compare( edgeTimestamp, sourceTimestamp ) < 1 );
final Long targetTimestamp = markedVersions.get( edge.getTargetNode() );
//the target Id has been marked for deletion. It's version is <= to the marked version for
// deletion,
// so we need to discard it
- if ( targetTimestamp != null && Long.compare( edgeTimestamp, targetTimestamp ) < 1 ) {
- return false;
+ final boolean isTargetDeleted = ( targetTimestamp != null && Long.compare( edgeTimestamp, targetTimestamp ) < 1 );
+
+ //one has been marked for deletion, return it
+ if(isSourceDeleted || isTargetDeleted){
+ return new SimpleMarkedEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(), edge.getTimestamp(), edge.isDeleted(), isSourceDeleted, isTargetDeleted );
}
+ return edge;
+ } ).filter( simpleMarkedEdge -> {
+ if(!filterMarked){
+ return true;
+ }
- return true;
- } );
+ //if any one of these is true, we filter it
+ return !(simpleMarkedEdge.isDeleted() || simpleMarkedEdge.isSourceNodeDelete() || simpleMarkedEdge.isTargetNodeDeleted());
+ });
} );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4e51d383/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
index 29d90eb..c6dc2e4 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
@@ -34,12 +34,23 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
private final boolean deleted;
+ private final boolean isSourceNodeDeleted;
+ private final boolean isTargetNodeDeleted;
public SimpleMarkedEdge( final Id sourceNode, final String type, final Id targetNode, final long timestamp, final boolean deleted) {
- super(sourceNode, type, targetNode, timestamp);
+ this( sourceNode, type, targetNode, timestamp, deleted, false, false );
+ }
+
+
+ public SimpleMarkedEdge( final Id sourceNode, final String type, final Id targetNode, final long timestamp,
+ final boolean deleted, final boolean isSourceNodeDeleted,
+ final boolean isTargetNodeDeleted ) {
+ super( sourceNode, type, targetNode, timestamp );
this.deleted = deleted;
+ this.isSourceNodeDeleted = isSourceNodeDeleted;
+ this.isTargetNodeDeleted = isTargetNodeDeleted;
}
@@ -56,6 +67,18 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
@Override
+ public boolean isSourceNodeDelete() {
+ return isSourceNodeDeleted;
+ }
+
+
+ @Override
+ public boolean isTargetNodeDeleted() {
+ return isTargetNodeDeleted;
+ }
+
+
+ @Override
public boolean equals( final Object o ) {
if ( this == o ) {
return true;
@@ -81,6 +104,8 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
public int hashCode() {
int result = super.hashCode();
result = 31 * result + ( deleted ? 1 : 0 );
+ result = 31 * result + ( isSourceNodeDeleted ? 1 : 0 );
+ result = 31 * result + ( isTargetNodeDeleted ? 1 : 0 );
return result;
}
@@ -88,8 +113,10 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
@Override
public String toString() {
return "SimpleMarkedEdge{" +
- "deleted=" + deleted +
- "} " + super.toString();
+ "deleted=" + deleted +
+ ", isSourceNodeDeleted=" + isSourceNodeDeleted +
+ ", isTargetNodeDeleted=" + isTargetNodeDeleted +
+ "} " + super.toString();
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4e51d383/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index 4b47bc0..4e4648e 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -70,7 +70,7 @@ limitations under the License.
<junit.version>4.11</junit.version>
<kryo-serializers.version>0.26</kryo-serializers.version>
<log4j.version>1.2.17</log4j.version>
- <rx.version>1.0.12</rx.version>
+ <rx.version>1.0.14</rx.version>
<slf4j.version>1.7.2</slf4j.version>
<surefire.version>2.16</surefire.version>
<aws.version>1.10.6</aws.version>
[48/50] [abbrv] usergrid git commit: Update the multi-tenant
migration script so we only have to use superuser.
Posted by to...@apache.org.
Update the multi-tenant migration script so we only have to use superuser.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/eb2756b7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/eb2756b7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/eb2756b7
Branch: refs/heads/master
Commit: eb2756b75ec06638cf50d5619b605fb312a62093
Parents: 40dfa06
Author: Michael Russo <mi...@gmail.com>
Authored: Fri Oct 30 13:18:32 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Fri Oct 30 13:18:32 2015 -0700
----------------------------------------------------------------------
stack/scripts/multitenant_migrate.py | 73 ++++++++++++-------------------
1 file changed, 27 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/eb2756b7/stack/scripts/multitenant_migrate.py
----------------------------------------------------------------------
diff --git a/stack/scripts/multitenant_migrate.py b/stack/scripts/multitenant_migrate.py
index 5f1918e..d1131c1 100644
--- a/stack/scripts/multitenant_migrate.py
+++ b/stack/scripts/multitenant_migrate.py
@@ -20,9 +20,9 @@
#
# STEP 1 - SETUP TENANT ONE TOMCAT RUNNING 2.1 NOT IN SERVICE AND INIT MIGRATION
#
-# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization:
#
-# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --init
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --init
#
# This command will setup and bootstrap the database, setup the migration system and update index mappings:
# - /system/database/setup
@@ -39,7 +39,7 @@
#
# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
#
-# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
+# python migrate_entity_data.py --org <org1name> --super <user>:<pass> --date <timestamp>
#
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app with a start-date specified so only data modified since
@@ -47,9 +47,9 @@
#
# STEP 3 - SETUP TENANT TWO TOMCAT RUNNING 2.1 NOT IN SERVICE
#
-# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization admin creds:
+# Login to the Tomcat instance and run this command, specifying both superuser and tenant organization:
#
-# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass>
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass>
#
# This command will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app.
@@ -60,17 +60,17 @@
#
# On the same Tomcat instance and run this command with the --date timestamp you noted in the previous step:
#
-# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --admin <user>:<pass> --date <timestamp>
+# python migrate_entity_data.py --org <org2name> --super <user>:<pass> --date <timestamp>
#
# Then it will migrate appinfos, re-index the management app and then for each of the specified org's apps
# it will de-dup connections and re-index the app with a start-date specified so only data modified since
# STEP 1 will be re-indexed.
#
-# STEP 5 - FULL DATA MIGRATION
+# STEP 5 - FULL DATA MIGRATION (migrates entity data to new format)
#
-# Login to any Tomcat instance in the cluster and run this command (admin user creds must be specificed but will be ignored):
+# Login to any Tomcat instance in the cluster and run this command:
#
-# python migrate_entity_data.py --super <user>:<pass> --admin <user>:<pass> --full
+# python migrate_entity_data.py --super <user>:<pass> --full
#
# This command will run the full data migration.
#
@@ -118,11 +118,6 @@ def parse_args():
type=str,
required=True)
- parser.add_argument('--admin',
- help='Organization admin creds <user:pass>',
- type=str,
- required=True)
-
parser.add_argument('--init',
help='Init system and start first migration.',
action='store_true',
@@ -154,14 +149,6 @@ def parse_args():
arg_vars['superuser'] = creds[0]
arg_vars['superpass'] = creds[1]
- creds = arg_vars['super'].split(':')
- if len(creds) != 2:
- print('Org admin credentials not properly specified. Must be "-u <user:pass>". Exiting...')
- exit_on_error()
- else:
- arg_vars['adminuser'] = creds[0]
- arg_vars['adminpass'] = creds[1]
-
return arg_vars
@@ -180,8 +167,6 @@ class Migrate:
self.logger = init_logging(self.__class__.__name__)
self.super_user = self.args['superuser']
self.super_pass = self.args['superpass']
- self.admin_user = self.args['adminuser']
- self.admin_pass = self.args['adminpass']
self.org = self.args['org']
self.init = self.args['init']
self.full = self.args['full']
@@ -220,7 +205,6 @@ class Migrate:
return
-
if self.init:
# Init the migration system as this is the first migration done on the cluster
@@ -250,7 +234,6 @@ class Migrate:
if index_mapping_updated:
break
-
# Migrate app info
if self.is_appinfo_migrated():
@@ -271,7 +254,6 @@ class Migrate:
break
self.logger.info('AppInfo Migration Ended.')
-
# Reindex management app
job = self.start_app_reindex(MANAGEMENT_APP_ID)
@@ -287,7 +269,6 @@ class Migrate:
self.logger.info("Finished Re-index. Job=[%s]", job)
self.metrics['reindex_end'] = get_current_time()
-
# Dedup and re-index all of organization's apps
app_ids = self.get_app_ids()
@@ -323,7 +304,6 @@ class Migrate:
self.log_metrics()
self.logger.info("Finished...")
-
except KeyboardInterrupt:
self.log_metrics()
self.logger.error('Keyboard interrupted migration. Please run again to ensure the migration finished.')
@@ -362,7 +342,7 @@ class Migrate:
def start_core_data_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -371,7 +351,7 @@ class Migrate:
def start_fulldata_migration(self):
try:
- r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=self.get_migration_url(), auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -382,7 +362,7 @@ class Migrate:
try:
# TODO fix this URL
migrateUrl = self.get_migration_url() + '/' + PLUGIN_MIGRATION_SYSTEM
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.super_pass))
+ r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -424,7 +404,7 @@ class Migrate:
def start_appinfo_migration(self):
try:
migrateUrl = self.get_migration_url() + '/' + PLUGIN_APPINFO
- r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ r = requests.put(url=migrateUrl, auth=(self.super_user, self.super_pass))
response = r.json()
return response
except requests.exceptions.RequestException as e:
@@ -639,24 +619,25 @@ class Migrate:
try:
- url = self.endpoint + "/management/token"
- body = json.dumps({"grant_type":"password","username":self.admin_user,"password":self.admin_pass})
- r = requests.post(url=url, data=body)
- if ( r.status_code != 200 ):
- print "Error logging in: " + r.text
- return
-
- access_token = r.json()["access_token"]
+ url = self.endpoint + "/management/organizations"
+ r = requests.get(url=url, auth=(self.super_user, self.super_pass))
- url = self.endpoint + "/management/orgs/" + self.org + "/apps?access_token=" + access_token
- r = requests.get(url=url)
if r.status_code != 200:
exit_on_error('Cannot get app ids: ' + r.text)
- apps = r.json()["data"]
+ response_json = r.json()
+
app_ids = []
- for appId in apps.values():
- app_ids.append(appId)
+ orgs = response_json["organizations"]
+ if orgs is not None:
+ for org in orgs:
+ if org["name"] == self.org:
+ for app in org["applications"]:
+ app_ids.append(org["applications"][app])
+ else:
+ e = 'No Orgs in this system'
+ self.logger.error(e)
+ exit_on_error(e)
return app_ids