You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/08/12 19:48:27 UTC
[1/3] incubator-usergrid git commit: Ensure we migrate app infos
before re-index when using the force (delta) option.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-dev 0794d9416 -> e3a4a9571
Ensure we migrate app infos before re-index when using the force (delta) option.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7476ea1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7476ea1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7476ea1b
Branch: refs/heads/two-dot-o-dev
Commit: 7476ea1b3c24033b733d556b94e3bd23d5d5a533
Parents: 9f81d7b
Author: Michael Russo <mi...@gmail.com>
Authored: Tue Aug 11 16:44:08 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Tue Aug 11 16:44:08 2015 -0700
----------------------------------------------------------------------
stack/scripts/migrate_entity_data.py | 42 +++++++++++++++++++++++++++----
1 file changed, 37 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7476ea1b/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index 40013ad..58b2b82 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -108,6 +108,14 @@ class Migrate:
self.reset_data_migration()
time.sleep(STATUS_INTERVAL_SECONDS)
self.is_data_migrated()
+ self.start_appinfo_migration()
+ self.logger.info('AppInfo Migration Started.')
+ is_appinfo_migrated = False
+ while not is_appinfo_migrated:
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ is_appinfo_migrated = self.is_appinfo_migrated()
+ if is_appinfo_migrated:
+ break
else:
self.logger.error('Entity Data has already been migrated. To re-run data migration provide the'
' force parameter: python migrate.py -u <user:pass> -f')
@@ -128,9 +136,9 @@ class Migrate:
self.metrics['reindex_end'] = get_current_time()
if not reindex_only:
- self.start_data_migration()
+ self.start_fulldata_migration()
self.metrics['data_migration_start'] = get_current_time()
- self.logger.info("Entity Data Migration Started")
+ self.logger.info("Full Data Migration Started")
is_migrated = False
while not is_migrated:
time.sleep(STATUS_INTERVAL_SECONDS)
@@ -164,7 +172,7 @@ class Migrate:
url = self.endpoint + '/system/index/rebuild'
return url
- def start_data_migration(self):
+ def start_fulldata_migration(self):
try:
r = requests.put(url=self.get_migration_url(), auth=(self.admin_user, self.admin_pass))
response = r.json()
@@ -173,6 +181,16 @@ class Migrate:
self.logger.error('Failed to start migration, %s', e)
exit_on_error(str(e))
+ def start_appinfo_migration(self):
+ try:
+ migrateUrl = self.get_migration_url() + '/' + 'appinfo-migration'
+ r = requests.put(url=migrateUrl, auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to start migration, %s', e)
+ exit_on_error(str(e))
+
def reset_data_migration(self):
version = TARGET_VERSION - 1
body = json.dumps({'collections-entity-data': version, 'appinfo-migration': version})
@@ -193,13 +211,27 @@ class Migrate:
appinfo_version = status['data']['appinfo-migration']
if entity_version == TARGET_VERSION and appinfo_version == TARGET_VERSION:
- self.logger.info('Data Migration status=[COMPLETE], collections-entity-data=[v%s], '
+ self.logger.info('Full Data Migration status=[COMPLETE], collections-entity-data=[v%s], '
'appinfo-migration=[v%s]',
entity_version,
appinfo_version)
return True
else:
- self.logger.info('Data Migration status=[NOTSTARTED/INPROGRESS]')
+ self.logger.info('Full Data Migration status=[NOTSTARTED/INPROGRESS]')
+ return False
+
+ def is_appinfo_migrated(self):
+ status = self.check_data_migration_status()
+ if status is not None:
+ appinfo_version = status['data']['appinfo-migration']
+
+ if appinfo_version == TARGET_VERSION:
+ self.logger.info('AppInfo Migration status=[COMPLETE],'
+ 'appinfo-migration=[v%s]',
+ appinfo_version)
+ return True
+ else:
+ self.logger.info('AppInfo Migration status=[NOTSTARTED/INPROGRESS]')
return False
def check_data_migration_status(self):
[3/3] incubator-usergrid git commit: Merge branch
'update-migrate-script' of
https://github.com/michaelarusso/incubator-usergrid into two-dot-o-dev
Posted by to...@apache.org.
Merge branch 'update-migrate-script' of https://github.com/michaelarusso/incubator-usergrid into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e3a4a957
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e3a4a957
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e3a4a957
Branch: refs/heads/two-dot-o-dev
Commit: e3a4a95718854da6aca7336884c3cb889fc5c770
Parents: 0794d94 5f64a59
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Aug 12 11:48:22 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Aug 12 11:48:22 2015 -0600
----------------------------------------------------------------------
stack/scripts/migrate_entity_data.py | 120 ++++++++++++++++++++++--------
1 file changed, 89 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-usergrid git commit: Slight change to migration
strategy and what happens on a delta migration.
Posted by to...@apache.org.
Slight change to migration strategy and what happens on a delta migration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5f64a59d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5f64a59d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5f64a59d
Branch: refs/heads/two-dot-o-dev
Commit: 5f64a59d2a25e60c0f5de8571704b50b6bebca10
Parents: 7476ea1
Author: Michael Russo <mi...@gmail.com>
Authored: Wed Aug 12 10:43:26 2015 -0700
Committer: Michael Russo <mi...@gmail.com>
Committed: Wed Aug 12 10:43:26 2015 -0700
----------------------------------------------------------------------
stack/scripts/migrate_entity_data.py | 94 ++++++++++++++++++++-----------
1 file changed, 60 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f64a59d/stack/scripts/migrate_entity_data.py
----------------------------------------------------------------------
diff --git a/stack/scripts/migrate_entity_data.py b/stack/scripts/migrate_entity_data.py
index 58b2b82..fd4d936 100644
--- a/stack/scripts/migrate_entity_data.py
+++ b/stack/scripts/migrate_entity_data.py
@@ -58,7 +58,7 @@ def parse_args():
required=True)
parser.add_argument('-f', '--force',
- help='Force the entity data migration to run. Used for delta data migrations.',
+ help='Force a delta migration.',
action='store_true',
default=False)
@@ -84,8 +84,10 @@ class Migrate:
self.endpoint = self.args['endpoint']
self.metrics = {'reindex_start': '',
'reindex_end': '',
- 'data_migration_start': '',
- 'data_migration_end': ''}
+ 'appinfo_migration_start': '',
+ 'appinfo_migration_end': '',
+ 'full_data_migration_start': '',
+ 'full_data_migration_end': ''}
self.logger = init_logging(self.__class__.__name__)
self.admin_user = self.args['user']
self.admin_pass = self.args['pass']
@@ -93,35 +95,35 @@ class Migrate:
def run(self):
self.logger.info('Initializing...')
- reindex_only = False
if not self.is_endpoint_available():
exit_on_error('Endpoint is not available, aborting')
if self.start_date is not None:
- self.logger.info("Date Provided. Only-Re-index will run from date=[%s]", self.start_date)
- reindex_only = True
- else:
- if self.is_data_migrated():
- if self.force_migration:
- self.logger.info('Force option provided.')
- self.reset_data_migration()
- time.sleep(STATUS_INTERVAL_SECONDS)
- self.is_data_migrated()
- self.start_appinfo_migration()
- self.logger.info('AppInfo Migration Started.')
- is_appinfo_migrated = False
- while not is_appinfo_migrated:
- time.sleep(STATUS_INTERVAL_SECONDS)
- is_appinfo_migrated = self.is_appinfo_migrated()
- if is_appinfo_migrated:
- break
- else:
- self.logger.error('Entity Data has already been migrated. To re-run data migration provide the'
- ' force parameter: python migrate.py -u <user:pass> -f')
- exit_on_error('Entity Data has already been migrated')
+ self.logger.info("Date Provided. Re-index will run from date=[%s]", self.start_date)
try:
+
+ # Always run an app info migration first
+ if self.is_appinfo_migrated():
+ self.logger.info('AppInfo already migrated. Resetting version for re-migration.')
+ self.reset_appinfo_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+
+ self.start_appinfo_migration()
+ self.logger.info('AppInfo Migration Started.')
+ self.metrics['appinfo_migration_start'] = get_current_time()
+
+ is_appinfo_migrated = False
+ while not is_appinfo_migrated:
+ is_appinfo_migrated = self.is_appinfo_migrated()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ if is_appinfo_migrated:
+ self.metrics['appinfo_migration_end'] = get_current_time()
+ break
+ self.logger.info('AppInfo Migration Ended.')
+
+ # Perform system re-index (it will grab date from input if provided)
job = self.start_reindex()
self.metrics['reindex_start'] = get_current_time()
self.logger.info('Started Re-index. Job=[%s]', job)
@@ -135,9 +137,17 @@ class Migrate:
self.logger.info("Finished Re-index. Job=[%s]", job)
self.metrics['reindex_end'] = get_current_time()
- if not reindex_only:
+ # Only when we do a delta (force migration) do we run the full data migration (includes entity data)
+ if self.force_migration:
+
+ self.logger.info('Force option provided. Performing full data migration...')
+ if self.is_data_migrated():
+ self.reset_data_migration()
+ time.sleep(STATUS_INTERVAL_SECONDS)
+ self.is_data_migrated()
+
self.start_fulldata_migration()
- self.metrics['data_migration_start'] = get_current_time()
+ self.metrics['full_data_migration_start'] = get_current_time()
self.logger.info("Full Data Migration Started")
is_migrated = False
while not is_migrated:
@@ -146,8 +156,8 @@ class Migrate:
if is_migrated:
break
- self.metrics['data_migration_end'] = get_current_time()
- self.logger.info("Entity Data Migration completed")
+ self.metrics['full_data_migration_end'] = get_current_time()
+ self.logger.info("Full Data Migration completed")
self.log_metrics()
self.logger.info("Finished...")
@@ -201,7 +211,19 @@ class Migrate:
'and appinfo-migration=[v%s]', version, version)
return response
except requests.exceptions.RequestException as e:
- self.logger.error('Failed to start migration, %s', e)
+ self.logger.error('Failed to reset full data migration versions, %s', e)
+ exit_on_error(str(e))
+
+ def reset_appinfo_migration(self):
+ version = TARGET_VERSION - 1
+ body = json.dumps({'appinfo-migration': version})
+ try:
+ r = requests.put(url=self.get_reset_migration_url(), data=body, auth=(self.admin_user, self.admin_pass))
+ response = r.json()
+ self.logger.info('Resetting appinfo migration versions to appinfo-migration=[v%s]', version)
+ return response
+ except requests.exceptions.RequestException as e:
+ self.logger.error('Failed to reset appinfo migration version, %s', e)
exit_on_error(str(e))
def is_data_migrated(self):
@@ -299,12 +321,16 @@ class Migrate:
self.logger.info(
'Re-index start=[%s], ' +
'Re-index end =[%s], ' +
- 'Entity Migration start=[%s], ' +
- 'Entity Migration end=[%s] ',
+ 'Full Data Migration start=[%s], ' +
+ 'Full Data Migration end=[%s] ' +
+ 'AppInfo Migration start=[%s], ' +
+ 'AppInfo Migration end=[%s] ',
self.metrics['reindex_start'],
self.metrics['reindex_end'],
- self.metrics['data_migration_start'],
- self.metrics['data_migration_end']
+ self.metrics['full_data_migration_start'],
+ self.metrics['full_data_migration_end'],
+ self.metrics['appinfo_migration_start'],
+ self.metrics['appinfo_migration_end']
)