You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/08/01 16:53:36 UTC
[01/50] [abbrv] usergrid git commit: Merge branch
'apigee-sso-provider' of https://github.com/apache/usergrid into
SSO2-Usergrid
Repository: usergrid
Updated Branches:
refs/heads/master 1e11464c7 -> 1d65496c0
Merge branch 'apigee-sso-provider' of https://github.com/apache/usergrid into SSO2-Usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/87dad497
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/87dad497
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/87dad497
Branch: refs/heads/master
Commit: 87dad4977d5541ebe31ef120eba399b2eb17f9b1
Parents: 6756035 6c36882
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Fri Jun 24 17:03:10 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Fri Jun 24 17:03:10 2016 -0700
----------------------------------------------------------------------
stack/README.md | 2 +-
.../main/resources/usergrid-default.properties | 8 +-
.../rest/management/ManagementResource.java | 56 +++-
.../security/sso/ApigeeSSO2Provider.java | 167 ++++++++++++
.../security/sso/ExternalSSOProvider.java | 22 ++
.../security/sso/SSOProviderFactory.java | 57 ++++
.../security/sso/UsergridExternalProvider.java | 263 +++++++++++++++++++
.../tokens/cassandra/TokenServiceImpl.java | 74 ++++--
.../externalProviders/ApigeeSSO2Provider.java | 108 --------
.../ExternalTokenProvider.java | 17 --
.../externalProviders/UsergridCentral.java | 259 ------------------
.../resources/usergrid-services-context.xml | 13 +-
12 files changed, 624 insertions(+), 422 deletions(-)
----------------------------------------------------------------------
[37/50] [abbrv] usergrid git commit: additional cleanup
Posted by mr...@apache.org.
additional cleanup
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3bc3d78b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3bc3d78b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3bc3d78b
Branch: refs/heads/master
Commit: 3bc3d78b5fb407c914b2e7040a686d0d2cbc20b1
Parents: 0952603
Author: Jeff West <jw...@apigee.com>
Authored: Tue Jul 26 16:58:51 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Tue Jul 26 16:58:51 2016 -0700
----------------------------------------------------------------------
.../es_tools/cluster_shard_allocation.py | 15 +-
.../es_tools/command_sender.py | 13 +-
.../es_tools/index_deleter.py | 20 +-
.../es_tools/index_iterator_size_checker.py | 285 -------------------
.../es_tools/index_replica_setter.py | 1 +
.../es_tools/index_shard_allocator.py | 22 +-
.../es_tools/mapping_retriever.py | 2 +-
.../es_tools/monitor_tasks.py | 1 +
.../general/duplicate_name_checker.py | 1 +
.../usergrid_tools/general/user_creator.py | 68 -----
.../iterators/usergrid_iterator.py | 2 +-
.../migration/usergrid_data_exporter.py | 25 +-
.../migration/usergrid_data_migrator.py | 9 +-
13 files changed, 53 insertions(+), 411 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
index 2e0fcbd..2b72640 100644
--- a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
+++ b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
@@ -24,26 +24,30 @@ import requests
__author__ = 'Jeff West @ ApigeeCorporation'
# The purpose of this script is to set certain nodes in an ElasticSearch cluster to be excluded from index allocation,
-# generally for the purpose of shutting down or restarting the node
+# generally for the purpose of decomissioning or troubleshooting a node.
+# you can optionally shut down nodes as they have all the replicas removed from them
SHUTDOWN_NODES = True
-nodes = [
+# these are the nodes which will have shard allocation disabled. The replicas will then be gradually moved off these
+# nodes. The amount of time required depends on the size of the index, speed of network, CPU and cluster load
+
+exclude_nodes = [
'elasticsearch206west',
'elasticsearch207west',
]
base_url = 'http://localhost:9200'
-exclude_nodes = nodes
-
nodes_string = ",".join(exclude_nodes)
print 'Excluding: ' + nodes_string
+
url_template = '%s/_cluster/settings' % base_url
status_code = 503
+# when a cluster is under load, it is possible that a 5xx will be returned.
while status_code >= 500:
r = requests.put(
'%s/_cluster/settings' % base_url,
@@ -62,7 +66,6 @@ ready = False
nodes_shut_down = []
while not ready:
-
ready = True
nodes_left = 0
bytes_left = 0
@@ -105,4 +108,4 @@ while not ready:
print 'NOT READY! Waiting for %s nodes and %s GB' % (nodes_left, bytes_left / 1024.0 / 1000000)
time.sleep(10)
-# print 'READY TO MOVE!'
+print 'READY TO MOVE!'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/es_tools/command_sender.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/command_sender.py b/utils/usergrid-util-python/es_tools/command_sender.py
index 8208e78..c2a5797 100644
--- a/utils/usergrid-util-python/es_tools/command_sender.py
+++ b/utils/usergrid-util-python/es_tools/command_sender.py
@@ -25,20 +25,11 @@ __author__ = 'Jeff West @ ApigeeCorporation'
# Simple utility to send commands, useful to not have to recall the proper format
-#
-# url = 'http://localhost:9200/_cat/shards'
-#
-# r = requests.get(url)
-#
-# response = r.text
-#
-# print response
-
data = {
"commands": [
{
"move": {
- "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
+ "index": "usergrid__APPID__application_target_final",
"shard": 14,
"from_node": "elasticsearch018",
"to_node": "elasticsearch021"
@@ -46,7 +37,7 @@ data = {
},
{
"move": {
- "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
+ "index": "usergrid__APPID__application_target_final",
"shard": 12,
"from_node": "elasticsearch018",
"to_node": "elasticsearch009"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/es_tools/index_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_deleter.py b/utils/usergrid-util-python/es_tools/index_deleter.py
index 9b60006..ac21a6f 100644
--- a/utils/usergrid-util-python/es_tools/index_deleter.py
+++ b/utils/usergrid-util-python/es_tools/index_deleter.py
@@ -16,6 +16,7 @@
# * specific language governing permissions and limitations
# * under the License.
# */
+from multiprocessing import Pool
import requests
import logging
@@ -58,6 +59,14 @@ counter = 0
process = False
delete_counter = 0
+indexes_to_delete = []
+
+
+def delete_index(index_name):
+ url_template = '%s/%s' % (url_base, index_name)
+ print 'DELETING Index [%s] %s at URL %s' % (delete_counter, index_name, url_template)
+ response = requests.delete('%s/%s' % (url_base, index))
+
for index in indices:
process = False
counter += 1
@@ -78,9 +87,12 @@ for index in indices:
process = False
if process:
- delete_counter += 1
+ indexes_to_delete.append(index)
+
+print 'Found [%s] indexes to delete: %s' % (len(indexes_to_delete), indexes_to_delete)
- url_template = '%s/%s' % (url_base, index)
- print 'DELETING Index [%s] %s at URL %s' % (delete_counter, index, url_template)
+if len(indexes_to_delete) > 0:
+ pool = Pool(4)
+ pool.map(delete_index, indexes_to_delete)
- response = requests.delete('%s/%s' % (url_base, index))
+print 'Done!'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
deleted file mode 100644
index 03924b2..0000000
--- a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
+++ /dev/null
@@ -1,285 +0,0 @@
-# */
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing,
-# * software distributed under the License is distributed on an
-# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# * KIND, either express or implied. See the License for the
-# * specific language governing permissions and limitations
-# * under the License.
-# */
-
-import json
-import re
-import traceback
-from multiprocessing.pool import Pool
-import requests
-
-index_url_template = 'http://localhost:9200/{index_name}/_search?size={size}&from={from_var}&q=-edgeName:zzzcollzzz|logs'
-
-index_names = [
- 'es-index-name'
-]
-
-baas_url = 'http://localhost:8080/org/{app_id}/{collection}/{entity_id}'
-
-field_part_map = {
- 'mockdata': 'mockData'
-}
-
-
-def update_entity_field(entity, field_name, field_value):
- entity_copy = entity.copy()
-
- worked = True
- is_array = False
- array_length = 0
-
- try:
- parts = field_name.split('.')
-
- if parts[len(parts) - 1] != 'size':
- print parts
- exit()
-
- change_me = entity_copy
-
- for i, field_part in enumerate(parts):
- field_part = field_part_map.get(field_part, field_part)
-
- if field_part == 'size':
- break
-
- if isinstance(change_me, dict):
- if field_part not in change_me:
- worked = False
- # print 'ERROR! field [%s] not in entity: %s' % (field_part, json.dumps(change_me))
- break
-
- change_me = change_me[field_part]
-
- elif isinstance(change_me, list):
- array_length = len(change_me)
-
- if i == len(parts) - 2 and len(parts) > i + 1 and parts[i + 1] == 'size':
-
- for j in xrange(0, len(change_me)):
- print 'arrau!'
- change_me[j] = update_entity_field(change_me[j], '.'.join(parts[i:]), field_value)
- # element['size'] = field_value
-
- elif len(change_me) == 1:
- print 'single array'
- change_me = change_me[0][field_part]
- else:
- print 'WTF!'
- try:
- change_me['size'] = field_value
- except:
- if array_length != 1:
- print traceback.format_exc()
- print 'damn'
-
- except:
- print '---Error updating field [%s] in document: %s' % (field_name, json.dumps(entity))
- print traceback.format_exc()
-
- if array_length > 1:
- print '++++++++ARRAY!!!!! %s' % array_length
-
- return entity_copy
-
-
-def update_entity_fields(entity, fields):
- entity_copy = entity.copy()
-
- for field in fields:
- field_name = field.get('name')
-
- if 'string' in field:
- field_value = field.get('string')
-
- elif 'long' in field:
- field_value = field.get('long')
-
- else:
- print 'Unexpected field type! %s' % json.dumps(field)
- return entity_copy
-
- entity_copy = update_entity_field(entity_copy, field_name, field_value)
-
- return entity_copy
-
-
-my = {
- 'foo': {
- 'bar': {
- 'color': 'red'
- }
- }
-}
-
-fields = [
- {
- 'name': 'foo.size',
- 'string': '2'
- },
- {
- 'name': 'foo.bar.size',
- 'long': 2
- }
-]
-
-
-def work(item):
- try:
- url = 'http://localhost:8080/org/{app_id}/{collection}/{entity_id}'.format(
- app_id=item[0],
- collection=item[1],
- entity_id=item[2]
- )
- print url
- r_get = requests.get(url)
-
- if r_get.status_code != 200:
- print 'ERROR GETTING ENTITY AT URL: %s' % url
- return
-
- response_json = r_get.json()
-
- entities = response_json.get('entities')
-
- if len(entities) <= 0:
- print 'TOO MANY ENTITIES AT URL: %s' % url
- return
-
- entity = entities[0]
-
- new_entity = update_entity_fields(entity, item[3])
-
- with open('/Users/ApigeeCorporation/tmp/hack/%s.json' % item[2], 'w') as f:
- json.dump(entity, f, indent=2)
-
- with open('/Users/ApigeeCorporation/tmp/hack/%s_new.json' % item[2], 'w') as f:
- json.dump(new_entity, f, indent=2)
-
- r_put = requests.put(url, data=json.dumps(new_entity))
-
- if r_put.status_code == 200:
- print 'PUT [%s]: %s' % (r_put.status_code, url)
- pass
- elif r_put.status_code:
- print 'PUT [%s]: %s | %s' % (r_put.status_code, url, r_put.text)
-
- except:
- print traceback.format_exc()
-
-
-POOL_SIZE = 4
-
-counter = 0
-size = POOL_SIZE * 10
-size = 1000
-
-total_docs = 167501577
-start_from = 0
-from_var = 0
-page = 0
-
-work_items = []
-
-pool = Pool(POOL_SIZE)
-
-keep_going = True
-
-while keep_going:
- work_items = []
-
- if from_var > total_docs:
- keep_going = False
- break
-
- from_var = start_from + (page * size)
- page += 1
-
- for index_name in index_names:
-
- index_url = index_url_template.format(index_name=index_name, size=size, from_var=from_var)
-
- print 'Getting URL: ' + index_url
-
- r = requests.get(index_url)
-
- if r.status_code != 200:
- print r.text
- exit()
-
- response = r.json()
-
- hits = response.get('hits', {}).get('hits')
-
- re_app_id = re.compile('appId\((.+),')
- re_ent_id = re.compile('entityId\((.+),')
- re_type = re.compile('entityId\(.+,(.+)\)')
-
- print 'Index: %s | hits: %s' % (index_name, len(hits))
-
- if len(hits) == 0:
- keep_going = False
- break
-
- for hit_data in hits:
- source = hit_data.get('_source')
-
- application_id = source.get('applicationId')
-
- app_id_find = re_app_id.findall(application_id)
-
- if len(app_id_find) > 0:
- app_id = app_id_find[0]
-
- entity_id_tmp = source.get('entityId')
-
- entity_id_find = re_ent_id.findall(entity_id_tmp)
- entity_type_find = re_type.findall(entity_id_tmp)
-
- if len(entity_id_find) > 0 and len(entity_type_find) > 0:
- entity_id = entity_id_find[0]
- collection = entity_type_find[0]
- fields_to_update = []
-
- for field in source.get('fields'):
- if field.get('name')[-5:] == '.size':
- fields_to_update.append(field)
-
- print json.dumps(source)
-
- work_items.append((app_id, collection, entity_id, fields_to_update))
-
- counter += 1
-
- print 'Work Items: %s' % len(work_items)
-
- try:
- pool.map(work, work_items)
-
-
- except:
- print traceback.format_exc()
-
- try:
- pool.map(work, work_items)
- except:
- pass
-
- print 'Work Done!'
-
-print 'done: %s' % counter
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/es_tools/index_replica_setter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_replica_setter.py b/utils/usergrid-util-python/es_tools/index_replica_setter.py
index 1214e48..383c195 100644
--- a/utils/usergrid-util-python/es_tools/index_replica_setter.py
+++ b/utils/usergrid-util-python/es_tools/index_replica_setter.py
@@ -103,6 +103,7 @@ def update_shards(index_name):
current_replicas = int(index_settings.get('number_of_replicas'))
if current_replicas == NUMBER_VALUE:
+ # no action required
return
success = False
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_shard_allocator.py b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
index d411744..d7f52f5 100644
--- a/utils/usergrid-util-python/es_tools/index_shard_allocator.py
+++ b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
@@ -32,31 +32,13 @@ __author__ = 'Jeff West @ ApigeeCorporation'
nodes_c32xl = [
'elasticsearch000eu',
'elasticsearch001eu',
- 'elasticsearch002eu',
- 'elasticsearch003eu',
- 'elasticsearch004eu',
- 'elasticsearch005eu',
- 'elasticsearch009eu',
- 'elasticsearch010eu',
- 'elasticsearch011eu',
- 'elasticsearch012eu',
- 'elasticsearch013eu',
- 'elasticsearch014eu',
+ 'elasticsearch002eu'
]
nodes_c34xl = [
'elasticsearch015eu',
'elasticsearch018eu',
- 'elasticsearch019eu',
- 'elasticsearch020eu',
- 'elasticsearch021eu',
- 'elasticsearch022eu',
- 'elasticsearch023eu',
- 'elasticsearch024eu',
- 'elasticsearch025eu',
- 'elasticsearch026eu',
- 'elasticsearch027eu',
- 'elasticsearch028eu'
+ 'elasticsearch019eu'
]
nodes = nodes_c34xl
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/es_tools/mapping_retriever.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_retriever.py b/utils/usergrid-util-python/es_tools/mapping_retriever.py
index 29fbe11..d3a55f0 100644
--- a/utils/usergrid-util-python/es_tools/mapping_retriever.py
+++ b/utils/usergrid-util-python/es_tools/mapping_retriever.py
@@ -49,7 +49,7 @@ for type_name, mapping_detail in mappings.iteritems():
print 'Processing %s' % type_name
- filename = '/Users/ApigeeCorporation/tmp/%s_%s_source_mapping.json' % (
+ filename = '/tmp/%s_%s_source_mapping.json' % (
SOURCE_INDEX, type_name)
print filename
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/es_tools/monitor_tasks.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/monitor_tasks.py b/utils/usergrid-util-python/es_tools/monitor_tasks.py
index b444322..8db30a0 100644
--- a/utils/usergrid-util-python/es_tools/monitor_tasks.py
+++ b/utils/usergrid-util-python/es_tools/monitor_tasks.py
@@ -25,6 +25,7 @@ __author__ = 'Jeff West @ ApigeeCorporation'
# Utility for monitoring pending tasks in ElasticSearch
+
def total_milliseconds(td):
return (td.microseconds + td.seconds * 1000000) / 1000
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
index a40e097..6b23403 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
@@ -19,6 +19,7 @@
from usergrid import UsergridQueryIterator
+
### This iterates a collection using GRAPH and checks whether there are more than on entity with the same name
url = 'https://host/org/app/collection?access_token=foo&limit=1000'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
deleted file mode 100644
index ace64ee..0000000
--- a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# */
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing,
-# * software distributed under the License is distributed on an
-# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# * KIND, either express or implied. See the License for the
-# * specific language governing permissions and limitations
-# * under the License.
-# */
-
-import json
-import requests
-
-### This will create an array of org-level management users
-
-users = [
- 'me@example.com'
-]
-
-for user in users:
-
- post_body = {
- "username": user,
- "name": user,
- "email": user,
- "password": "test12345"
- }
-
- print json.dumps(post_body)
-
- r = requests.post('http://localhost:8080/management/organizations/asdf/users',
- headers={
- 'Authorization': 'Bearer SADFSDF',
- 'Content-Type': 'application/json'
- },
- data=json.dumps(post_body))
-
- print r.status_code
-
- print '%s: created (POST) [%s]: %s' % (user, r.status_code, r.text)
-
- #
- # r = requests.put('http://localhost:8080/management/users/%s' % user,
- # headers={
- # 'Authorization': 'Bearer YWMtFlVrhK8nEeW-AhmxdmpAVAAAAVIYTHxTNSUxpQyUWZQ2LsZxcXSdNtO_lWo',
- # 'Content-Type': 'application/json'
- # },
- # data=json.dumps('{"confirmed": true}'))
- #
- # print '%s: confirmed: %s' % (user, r.status_code)
- #
- # r = requests.put('http://localhost:8080/management/users/%s' % user,
- # headers={
- # 'Authorization': 'Bearer YWMtFlVrhK8nEeW-AhmxdmpAVAAAAVIYTHxTNSUxpQyUWZQ2LsZxcXSdNtO_lWo',
- # 'Content-Type': 'application/json'
- # },
- # data=json.dumps('{"activated": true}'))
- #
- # print '%s: activated: %s' % (user, r.status_code)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
index 04328ab..632aa68 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
@@ -246,7 +246,7 @@ def create_new(org_name, app_name, collection_name, entity_data, source_client,
e = c.entity_from_data(entity_data)
e.put()
- except UsergridError, err:
+ except UsergridError as err:
logger.error(err)
raise err
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
index e374fbc..edfa1c6 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
@@ -17,6 +17,7 @@
# * under the License.
# */
+from __future__ import print_function
import os
import uuid
from Queue import Empty
@@ -186,7 +187,7 @@ class StatusListener(Process):
org_results['apps'][app]['collections'].update(status_map)
try:
- for app, app_data in org_results['apps'].iteritems():
+ for app, app_data in org_results['apps'].items():
app_data['summary'] = {
'max_created': -1,
'max_modified': -1,
@@ -197,7 +198,7 @@ class StatusListener(Process):
}
if 'collections' in app_data:
- for collection, collection_data in app_data['collections'].iteritems():
+ for collection, collection_data in app_data['collections'].items():
app_data['summary']['count'] += collection_data['count']
app_data['summary']['bytes'] += collection_data['bytes']
@@ -236,13 +237,13 @@ class StatusListener(Process):
status_logger.warn('UPDATED status of org processed: %s' % json.dumps(org_results))
- except KeyboardInterrupt, e:
+ except KeyboardInterrupt as e:
raise e
except:
- print traceback.format_exc()
+ print(traceback.format_exc())
- except KeyboardInterrupt, e:
+ except KeyboardInterrupt as e:
status_logger.warn('FINAL status of org processed: %s' % json.dumps(org_results))
raise e
@@ -260,7 +261,7 @@ class StatusListener(Process):
keep_going = False
except:
- print traceback.format_exc()
+ print(traceback.format_exc())
logger.warn('FINAL status of org processed: %s' % json.dumps(org_results))
@@ -305,7 +306,7 @@ class EntityExportWorker(Process):
collection_worker_logger.info('Done! Finished app/collection: %s / %s' % (app, collection_name))
- except KeyboardInterrupt, e:
+ except KeyboardInterrupt as e:
raise e
except Empty:
@@ -316,9 +317,9 @@ class EntityExportWorker(Process):
if empty_count >= 2:
keep_going = False
- except Exception, e:
+ except Exception as e:
logger.exception('Error in CollectionWorker processing collection [%s]' % collection_name)
- print traceback.format_exc()
+ print(traceback.format_exc())
finally:
if entity_file is not None:
@@ -790,7 +791,7 @@ def main():
limit=config.get('limit'),
**config.get('source_endpoint'))
- print 'Retrieving apps from [%s]' % source_org_mgmt_url
+ print('Retrieving apps from [%s]' % source_org_mgmt_url)
logger.info('Retrieving apps from [%s]' % source_org_mgmt_url)
try:
@@ -807,9 +808,9 @@ def main():
org_apps = r.json().get('data')
- except Exception, e:
+ except Exception as e:
logger.exception('ERROR Retrieving apps from [%s]' % source_org_mgmt_url)
- print traceback.format_exc()
+ print(traceback.format_exc())
logger.critical('Unable to retrieve apps from [%s] and will exit' % source_org_mgmt_url)
exit()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bc3d78b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
index c99aa12..0ed0539 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
@@ -17,6 +17,9 @@
# * under the License.
# */
+from __future__ import print_function
+from __future__ import print_function
+from __future__ import print_function
import os
import uuid
from Queue import Empty
@@ -2022,7 +2025,7 @@ def filter_apps_and_collections(org_apps):
logger.info('App=[%s] filtered Collections=[%s]' % (app, collections))
except:
- print traceback.format_exc()
+ print(traceback.format_exc())
return app_collecitons
@@ -2113,7 +2116,7 @@ def main():
limit=config.get('limit'),
**config.get('source_endpoint'))
- print 'Retrieving apps from [%s]' % source_org_mgmt_url
+ print('Retrieving apps from [%s]' % source_org_mgmt_url)
logger.info('Retrieving apps from [%s]' % source_org_mgmt_url)
try:
@@ -2132,7 +2135,7 @@ def main():
except Exception:
logger.exception('ERROR Retrieving apps from [%s]' % source_org_mgmt_url)
- print traceback.format_exc()
+ print(traceback.format_exc())
logger.critical('Unable to retrieve apps from [%s] and will exit' % source_org_mgmt_url)
exit()
[36/50] [abbrv] usergrid git commit: cleaned up with decent sample
data
Posted by mr...@apache.org.
cleaned up with decent sample data
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/09526036
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/09526036
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/09526036
Branch: refs/heads/master
Commit: 09526036d90704819bf269d3f411295bd2fff47f
Parents: a57f865
Author: Jeff West <jw...@apigee.com>
Authored: Tue Jul 26 16:15:11 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Tue Jul 26 16:15:11 2016 -0700
----------------------------------------------------------------------
utils/usergrid-util-python/es_tools/alias_mover.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/09526036/utils/usergrid-util-python/es_tools/alias_mover.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/alias_mover.py b/utils/usergrid-util-python/es_tools/alias_mover.py
index 3b5caab..a372d14 100644
--- a/utils/usergrid-util-python/es_tools/alias_mover.py
+++ b/utils/usergrid-util-python/es_tools/alias_mover.py
@@ -23,13 +23,13 @@ import requests
cluster = 'cluster-1'
work = {
- # 'remove': {
- # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'applications_3',
- # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'applications_3',
- # },
+ 'remove': {
+ 'app-id-1': 'from-index',
+ 'app-id-2': 'from-index'
+ },
'add': {
- '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'my-index-1-no-doc-18',
- '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'my-index-1-no-doc-18',
+ 'app-id-1': 'to-index',
+ 'app-id-2': 'to-index'
}
}
[21/50] [abbrv] usergrid git commit: superuser to be able to create
an org with new user or existing user.
Posted by mr...@apache.org.
superuser to be able to create an org with new user or existing user.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5fcad445
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5fcad445
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5fcad445
Branch: refs/heads/master
Commit: 5fcad44504950f39be9d13e0d5224b04699f42c7
Parents: fad65a8
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Thu Jul 21 09:35:52 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Thu Jul 21 09:35:52 2016 -0700
----------------------------------------------------------------------
.../usergrid/rest/AbstractContextResource.java | 16 ++++++++++
.../rest/management/ManagementResource.java | 32 +++++++++++---------
.../organizations/OrganizationsResource.java | 6 ++--
stack/services/pom.xml | 2 +-
.../cassandra/ManagementServiceImpl.java | 27 +++++++++++++----
5 files changed, 58 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5fcad445/stack/rest/src/main/java/org/apache/usergrid/rest/AbstractContextResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/AbstractContextResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/AbstractContextResource.java
index 77d2d1e..fc95b19 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/AbstractContextResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/AbstractContextResource.java
@@ -55,6 +55,9 @@ public abstract class AbstractContextResource {
};
protected static final ObjectMapper mapper = new ObjectMapper();
+ public final static String ROLE_SERVICE_ADMIN = "service-admin";
+ public static final String USERGRID_SYSADMIN_LOGIN_NAME = "usergrid.sysadmin.login.name";
+
protected AbstractContextResource parent;
@@ -258,4 +261,17 @@ public abstract class AbstractContextResource {
}
return jsonObject;
}
+
+
+ /**
+ * check if its a system admin
+ * @return
+ */
+ public Boolean userServiceAdmin(String username) {
+
+ if (sc.isUserInRole(ROLE_SERVICE_ADMIN) || (username != null && username.equals(properties.getProperty(USERGRID_SYSADMIN_LOGIN_NAME)))) {
+ return true;
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5fcad445/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 77569d6..7217c49 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.rest.management;
import org.apache.amber.oauth2.common.error.OAuthError;
import org.apache.amber.oauth2.common.exception.OAuthProblemException;
+import org.apache.amber.oauth2.common.exception.OAuthSystemException;
import org.apache.amber.oauth2.common.message.OAuthResponse;
import org.apache.amber.oauth2.common.message.types.GrantType;
import org.apache.commons.lang.StringUtils;
@@ -269,6 +270,7 @@ public class ManagementResource extends AbstractContextResource {
@QueryParam( "client_id" ) String client_id,
@QueryParam( "client_secret" ) String client_secret, @QueryParam( "ttl" ) long ttl,
@QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
+
return getAccessTokenInternal( ui, authorization, grant_type, username, password, client_id, client_secret, ttl,
callback, false, false);
}
@@ -305,6 +307,7 @@ public class ManagementResource extends AbstractContextResource {
ensureAuthenticationAllowed( username, grant_type );
// }
+
if ( authorization != null ) {
String type = stringOrSubstringBeforeFirst( authorization, ' ' ).toUpperCase();
@@ -381,7 +384,8 @@ public class ManagementResource extends AbstractContextResource {
}
//moved the check for sso enabled form MangementServiceImpl since was unable to get the current user there to check if its super user.
- if( tokens.isExternalSSOProviderEnabled() && !user.getUsername().equals(properties.getProperty(USERGRID_SYSADMIN_LOGIN_NAME)) ){
+ if( tokens.isExternalSSOProviderEnabled()
+ && !userServiceAdmin(user.getUsername()) ){
throw new RuntimeException("SSO Integration is enabled, Admin users must login via provider: "+
properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER));
}
@@ -458,6 +462,8 @@ public class ManagementResource extends AbstractContextResource {
@QueryParam( "callback" ) @DefaultValue( "" ) String callback )
throws Exception {
+ ValidateJson(json);
+
String grant_type = ( String ) json.get( "grant_type" );
String username = ( String ) json.get( "username" );
String password = ( String ) json.get( "password" );
@@ -487,14 +493,7 @@ public class ManagementResource extends AbstractContextResource {
@HeaderParam( "Authorization" ) String authorization ) throws Exception {
- if ( json == null ) {
- String errorDescription = "invalid request, expected data in the request.";
- OAuthResponse response =
- OAuthResponse.errorResponse( SC_BAD_REQUEST ).setError( OAuthError.TokenResponse.INVALID_REQUEST )
- .setErrorDescription( errorDescription ).buildJSONMessage();
- return Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
- .entity( wrapWithCallback( response.getBody(), callback ) ).build();
- }
+ ValidateJson(json);
String grant_type = ( String ) json.get( "grant_type" );
String username = ( String ) json.get( "username" );
@@ -516,6 +515,12 @@ public class ManagementResource extends AbstractContextResource {
callback, false, false );
}
+ private void ValidateJson(Map<String, Object> json) throws OAuthSystemException {
+ if ( json == null ) {
+ throw new IllegalArgumentException("missing json post data");
+ }
+ }
+
@GET
@Path( "authorize" )
@@ -600,6 +605,7 @@ public class ManagementResource extends AbstractContextResource {
*/
private void ensureAuthenticationAllowed( String username, String grant_type ) {
+
if ( username == null || grant_type == null || !grant_type.equalsIgnoreCase( "password" )) {
return; // we only care about username/password auth
}
@@ -609,12 +615,8 @@ public class ManagementResource extends AbstractContextResource {
// !StringUtils.isEmpty( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
if ( tokens.isExternalSSOProviderEnabled() ) {
-
// when external tokens enabled then only superuser can obtain an access token
-
- final String superuserName = properties.getProperty( USERGRID_SYSADMIN_LOGIN_NAME );
- if ( !username.equalsIgnoreCase( superuserName )) {
-
+ if ( userServiceAdmin(username)) {
// this guy is not the superuser
throw new IllegalArgumentException( "Admin Users must login via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
@@ -623,6 +625,8 @@ public class ManagementResource extends AbstractContextResource {
}
+
+
String errorMsg = "";
String responseType;
String clientId;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5fcad445/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
index eb70486..823ebcc 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
@@ -28,7 +28,6 @@ import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.RootResource;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
-import org.apache.usergrid.security.shiro.utils.SubjectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -188,10 +187,9 @@ public class OrganizationsResource extends AbstractContextResource {
String email, String password, Map<String, Object> userProperties,
Map<String, Object> orgProperties, String callback ) throws Exception {
- String tokenUserName = SubjectUtils.getUser().getUsername();
-
if ( tokens.isExternalSSOProviderEnabled() ) {
- if(!tokenUserName.equals(properties.getProperty(USERGRID_SYSADMIN_LOGIN_NAME))) {
+ //let superuser add an org even if external SSO Provider is enabled.
+ if(!userServiceAdmin(null) ) { // what should the username be ?
throw new IllegalArgumentException("Organization / Admin Users must be created via " +
properties.getProperty(USERGRID_EXTERNAL_PROVIDER_URL));
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5fcad445/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index fdbd9c9..7a654f5 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -179,7 +179,7 @@
<dependency>
<groupId>io.jsonwebtoken</groupId>
<artifactId>jjwt</artifactId>
- <version>0.2</version>
+ <version>0.6.0</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5fcad445/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 90eb4c9..e8bf0ec 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -548,11 +548,25 @@ public class ManagementServiceImpl implements ManagementService {
if ( !validateAdminInfo( username, name, email, password ) ) {
return null;
}
- if ( areActivationChecksDisabled() ) {
- user = createAdminUserInternal( null, username, name, email, password, true, false, userProperties );
+
+ // irrespective of it being sso enabled or not , if its a super user request it will try to fetch user if no password is passed.
+ if(password == null && SubjectUtils.isServiceAdmin()){
+ user = getAdminUserByEmail(email);
+ if(user == null ){
+ throw new IllegalArgumentException("Password should be sent in the request or should be a valid admin user email.");
+ }
}
- else {
- user = createAdminUserInternal( null, username, name, email, password, activated, disabled, userProperties );
+ else if(password == null ){ //for existing workflow.
+ throw new IllegalArgumentException("Password should be sent in the request.");
+ }
+
+
+ if(user == null) {
+ if ((tokens.isExternalSSOProviderEnabled() && SubjectUtils.isServiceAdmin()) || areActivationChecksDisabled()) {
+ user = createAdminUserInternal(null, username, name, email, password, true, false, userProperties);
+ } else {
+ user = createAdminUserInternal(null, username, name, email, password, activated, disabled, userProperties);
+ }
}
if(logger.isTraceEnabled()){
@@ -964,11 +978,11 @@ public class ManagementServiceImpl implements ManagementService {
EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
- if ( !em.isPropertyValueUniqueForEntity( "user", "username", username ) ) {
+ if ( !( tokens.isExternalSSOProviderEnabled() && SubjectUtils.isServiceAdmin()) && !em.isPropertyValueUniqueForEntity( "user", "username", username ) ) {
throw new DuplicateUniquePropertyExistsException( "user", "username", username );
}
- if ( !em.isPropertyValueUniqueForEntity( "user", "email", email ) ) {
+ if ( !(tokens.isExternalSSOProviderEnabled()&& SubjectUtils.isServiceAdmin()) && !em.isPropertyValueUniqueForEntity( "user", "email", email ) ) {
throw new DuplicateUniquePropertyExistsException( "user", "email", email );
}
return true;
@@ -3479,4 +3493,5 @@ public class ManagementServiceImpl implements ManagementService {
scopedCache.invalidate();
localShiroCache.invalidateAll();
}
+
}
[15/50] [abbrv] usergrid git commit: changing the import to point to
the correct one.
Posted by mr...@apache.org.
changing the import to point to the correct one.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5c54d15e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5c54d15e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5c54d15e
Branch: refs/heads/master
Commit: 5c54d15e024229d99c78aa0ca5ae6c9beeab4430
Parents: 6a790a8
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Fri Jul 8 12:33:43 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Fri Jul 8 12:33:43 2016 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/security/sso/ApigeeSSO2Provider.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5c54d15e/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
index 9871cc7..953da55 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
@@ -20,7 +20,7 @@ import io.jsonwebtoken.*;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.management.exceptions.ExternalSSOProviderAdminUserNotFoundExceptions;
+import org.apache.usergrid.management.exceptions.ExternalSSOProviderAdminUserNotFoundException;
import org.apache.usergrid.security.AuthPrincipalInfo;
import org.apache.usergrid.security.AuthPrincipalType;
import org.apache.usergrid.security.tokens.TokenInfo;
@@ -82,7 +82,7 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
UserInfo userInfo = validateAndReturnUserInfo(token, ttl);
if(userInfo == null){
- throw new ExternalSSOProviderAdminUserNotFoundExceptions("Unable to load user from token: "+token);
+ throw new ExternalSSOProviderAdminUserNotFoundException("Unable to load user from token: "+token);
}
return new TokenInfo(UUIDUtils.newTimeUUID(), "access", 1, 1, 1, ttl,
[45/50] [abbrv] usergrid git commit: updated author
Posted by mr...@apache.org.
updated author
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ac9cfff1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ac9cfff1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ac9cfff1
Branch: refs/heads/master
Commit: ac9cfff184ec217a2c60389044c961ffc241bb2a
Parents: c47e6f6
Author: Jeff West <jw...@apigee.com>
Authored: Thu Jul 28 14:15:52 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Thu Jul 28 14:15:52 2016 -0700
----------------------------------------------------------------------
.../es_tools/alias_mover.py | 2 ++
.../es_tools/cluster_shard_allocation.py | 2 +-
.../es_tools/command_sender.py | 2 +-
.../es_tools/es_index_iterator_reindexer.py | 2 ++
.../es_tools/es_searcher.py | 2 +-
.../es_tools/index_deleter.py | 2 +-
.../es_tools/index_prefix_checker.py | 2 +-
.../es_tools/index_replica_setter.py | 2 +-
.../es_tools/index_shard_allocator.py | 2 +-
.../es_tools/mapping_deleter.py | 2 +-
.../es_tools/mapping_retriever.py | 2 +-
.../es_tools/monitor_tasks.py | 2 +-
.../index_test/document_creator.py | 3 +++
.../index_test/index_test_mixed_batch.py | 3 +++
.../index_test/index_test_single_type_batch.py | 3 +++
.../activity_streams/activity_streams.py | 3 +++
.../samples/beacon-event-example.py | 4 ++++
.../samples/counter_test.py | 3 +++
utils/usergrid-util-python/setup.py | 2 +-
.../usergrid_tools/general/__init__.py | 5 +++-
.../usergrid_tools/general/deleter.py | 2 +-
.../general/duplicate_name_checker.py | 2 ++
.../usergrid_tools/general/queue_monitor.py | 2 +-
.../usergrid_tools/general/url_tester.py | 5 +++-
.../general/user_confirm_activate.py | 5 +++-
.../usergrid_tools/groups/__init__.py | 2 ++
.../usergrid_tools/groups/big_group_creater.py | 3 +++
.../usergrid_tools/indexing/__init__.py | 5 +++-
.../usergrid_tools/indexing/batch_index_test.py | 3 +++
.../indexing/entity_index_test.py | 3 +++
.../usergrid_tools/iterators/simple_iterator.py | 3 +++
.../iterators/usergrid_cross_region_iterator.py | 2 +-
.../iterators/usergrid_iterator.py | 2 +-
.../usergrid_tools/library_check.py | 3 +++
.../usergrid_tools/migration/__init__.py | 24 +++++++++++++++++++-
.../migration/usergrid_data_exporter.py | 2 +-
.../migration/usergrid_data_migrator.py | 2 +-
.../usergrid_tools/parse_importer/__init__.py | 5 +++-
.../parse_importer/parse_importer.py | 2 +-
.../usergrid_tools/permissions/permissions.py | 3 +++
.../queue/dlq-iterator-checker.py | 2 +-
.../usergrid_tools/queue/dlq_requeue.py | 2 +-
.../usergrid_tools/queue/queue_cleaner.py | 2 +-
.../usergrid_tools/redis/redis_iterator.py | 3 +++
.../usergrid_tools/redis/redisscan.py | 3 +++
45 files changed, 115 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/alias_mover.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/alias_mover.py b/utils/usergrid-util-python/es_tools/alias_mover.py
index a372d14..bc2bfb7 100644
--- a/utils/usergrid-util-python/es_tools/alias_mover.py
+++ b/utils/usergrid-util-python/es_tools/alias_mover.py
@@ -20,6 +20,8 @@
import json
import requests
+__author__ = 'Jeff.West@yahoo.com'
+
cluster = 'cluster-1'
work = {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
index 2b72640..8f47698 100644
--- a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
+++ b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
@@ -21,7 +21,7 @@ import json
import time
import requests
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
# The purpose of this script is to set certain nodes in an ElasticSearch cluster to be excluded from index allocation,
# generally for the purpose of decomissioning or troubleshooting a node.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/command_sender.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/command_sender.py b/utils/usergrid-util-python/es_tools/command_sender.py
index c2a5797..2637ca4 100644
--- a/utils/usergrid-util-python/es_tools/command_sender.py
+++ b/utils/usergrid-util-python/es_tools/command_sender.py
@@ -20,7 +20,7 @@
import json
import requests
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
# Simple utility to send commands, useful to not have to recall the proper format
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
index 85872e2..101fa98 100644
--- a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
+++ b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
@@ -22,6 +22,8 @@ import re
from multiprocessing.pool import Pool
import requests
+__author__ = 'Jeff.West@yahoo.com'
+
# This script iterates an index and issues a PUT request for an empty string to force a reindex of the entity
index_url_template = 'http://elasticsearch013wo:9200/{index_name}/_search?size={size}&from={from_var}'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/es_searcher.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_searcher.py b/utils/usergrid-util-python/es_tools/es_searcher.py
index 4c7a297..5fb66dc 100644
--- a/utils/usergrid-util-python/es_tools/es_searcher.py
+++ b/utils/usergrid-util-python/es_tools/es_searcher.py
@@ -22,7 +22,7 @@ import requests
# Simple example of searching for a specific entity in ES
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
INDEX_NAME=''
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/index_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_deleter.py b/utils/usergrid-util-python/es_tools/index_deleter.py
index ac21a6f..cb69f00 100644
--- a/utils/usergrid-util-python/es_tools/index_deleter.py
+++ b/utils/usergrid-util-python/es_tools/index_deleter.py
@@ -21,7 +21,7 @@ from multiprocessing import Pool
import requests
import logging
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
# utility for deleting indexes that are no longer needed. Given:
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/index_prefix_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_prefix_checker.py b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
index 0ccb245..d6a5d70 100644
--- a/utils/usergrid-util-python/es_tools/index_prefix_checker.py
+++ b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
@@ -22,7 +22,7 @@ from collections import defaultdict
import requests
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
# This script iterates all the indexes in an ES cluster and aggregates the size by the prefix
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/index_replica_setter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_replica_setter.py b/utils/usergrid-util-python/es_tools/index_replica_setter.py
index 383c195..7f27d70 100644
--- a/utils/usergrid-util-python/es_tools/index_replica_setter.py
+++ b/utils/usergrid-util-python/es_tools/index_replica_setter.py
@@ -21,7 +21,7 @@ from multiprocessing import Pool
import requests
import time
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
# utility for updating the replicas of a set of indexes that are no longer needed. Given:
# A) a set of strings to include when evaluating the index names to update
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/index_shard_allocator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_shard_allocator.py b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
index d7f52f5..47e05e2 100644
--- a/utils/usergrid-util-python/es_tools/index_shard_allocator.py
+++ b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
@@ -22,7 +22,7 @@ from multiprocessing import Pool
import requests
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
# The purpose of this script is to update the shard allocation of ElasticSearch for specific indexes to be set to
# specific nodes. The reason for doing this is to isolate the nodes on which certain indexes run for specific
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/mapping_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_deleter.py b/utils/usergrid-util-python/es_tools/mapping_deleter.py
index 112cc20..a070d76 100644
--- a/utils/usergrid-util-python/es_tools/mapping_deleter.py
+++ b/utils/usergrid-util-python/es_tools/mapping_deleter.py
@@ -22,7 +22,7 @@ import json
import requests
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
url_base = 'http://localhost:9200'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/mapping_retriever.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_retriever.py b/utils/usergrid-util-python/es_tools/mapping_retriever.py
index d3a55f0..473f278 100644
--- a/utils/usergrid-util-python/es_tools/mapping_retriever.py
+++ b/utils/usergrid-util-python/es_tools/mapping_retriever.py
@@ -20,7 +20,7 @@
import json
import requests
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
# Utility to iterate the mappings for an index and save them locally
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/es_tools/monitor_tasks.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/monitor_tasks.py b/utils/usergrid-util-python/es_tools/monitor_tasks.py
index 8db30a0..7ceb61d 100644
--- a/utils/usergrid-util-python/es_tools/monitor_tasks.py
+++ b/utils/usergrid-util-python/es_tools/monitor_tasks.py
@@ -21,7 +21,7 @@ import datetime
import requests
import time
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
# Utility for monitoring pending tasks in ElasticSearch
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/index_test/document_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/document_creator.py b/utils/usergrid-util-python/index_test/document_creator.py
index a43e965..f13ccc2 100644
--- a/utils/usergrid-util-python/index_test/document_creator.py
+++ b/utils/usergrid-util-python/index_test/document_creator.py
@@ -31,6 +31,9 @@ import argparse
import loremipsum
+__author__ = 'Jeff.West@yahoo.com'
+
+
def parse_args():
parser = argparse.ArgumentParser(description='ElasticSearch Index Test 1')
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
index 3e6552e..99db41f 100644
--- a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
+++ b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
@@ -30,6 +30,9 @@ import loremipsum
import requests
from elasticsearch import Elasticsearch
+__author__ = 'Jeff.West@yahoo.com'
+
+
es_hosts = [
{'host': 'elasticsearch000west', 'port': 9200},
{'host': 'elasticsearch001west', 'port': 9200},
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
index cc25570..f5ee9d6 100644
--- a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
+++ b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
@@ -31,6 +31,9 @@ import loremipsum
import requests
from elasticsearch import Elasticsearch
+__author__ = 'Jeff.West@yahoo.com'
+
+
es_hosts = [
{'host': 'elasticsearch000west', 'port': 9200},
{'host': 'elasticsearch001west', 'port': 9200},
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
index 9d95bef..b838485 100644
--- a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
+++ b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
@@ -17,6 +17,9 @@
# * under the License.
# */
+__author__ = 'Jeff.West@yahoo.com'
+
+
# docs page: http://docs.apigee.com/api-baas/content/creating-activity
# create user 1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/samples/beacon-event-example.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/beacon-event-example.py b/utils/usergrid-util-python/samples/beacon-event-example.py
index 8863b35..8e8bf02 100644
--- a/utils/usergrid-util-python/samples/beacon-event-example.py
+++ b/utils/usergrid-util-python/samples/beacon-event-example.py
@@ -18,6 +18,10 @@
# */
# URL Templates for Usergrid
+
+__author__ = 'Jeff.West@yahoo.com'
+
+
import json
import random
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/samples/counter_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/counter_test.py b/utils/usergrid-util-python/samples/counter_test.py
index 4e393b7..f5276c4 100644
--- a/utils/usergrid-util-python/samples/counter_test.py
+++ b/utils/usergrid-util-python/samples/counter_test.py
@@ -22,6 +22,9 @@ import json
import requests
+__author__ = 'Jeff.West@yahoo.com'
+
+
tstamp = time.gmtime() * 1000
s = requests.Session()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/setup.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/setup.py b/utils/usergrid-util-python/setup.py
index 83ee6ae..337c914 100755
--- a/utils/usergrid-util-python/setup.py
+++ b/utils/usergrid-util-python/setup.py
@@ -19,7 +19,7 @@
from setuptools import setup, find_packages
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
VERSION = '0.5.13'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/general/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/__init__.py b/utils/usergrid-util-python/usergrid_tools/general/__init__.py
index b64a076..3b2a4e0 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/__init__.py
@@ -15,4 +15,7 @@
# * KIND, either express or implied. See the License for the
# * specific language governing permissions and limitations
# * under the License.
-# */
\ No newline at end of file
+# */
+
+__author__ = 'Jeff.West@yahoo.com'
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/general/deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/deleter.py b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
index 3e21fa4..a62b6e3 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/deleter.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
@@ -21,7 +21,7 @@ import json
import traceback
import requests
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
def total_milliseconds(td):
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
index 6b23403..6957290 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
@@ -19,6 +19,8 @@
from usergrid import UsergridQueryIterator
+__author__ = 'Jeff.West@yahoo.com'
+
### This iterates a collection using GRAPH and checks whether there are more than on entity with the same name
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
index 2ad2950..98420fb 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
@@ -30,7 +30,7 @@ from boto import sqs
### This monitors an SQS queue and measures the delta message count between polling intervals to infer the amount of time
### remaining to fully drain the queue
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
def total_seconds(td):
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
index 9d999d6..6e3bef8 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
@@ -22,7 +22,10 @@ import time
import numpy
import requests
-## This will call a URL over and over to check the latency of the call
+__author__ = 'Jeff.West@yahoo.com'
+
+# This will call a URL over and over to check the latency of the call
+
def total_milliseconds(td):
return (td.microseconds + td.seconds * 1000000) / 1000
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
index 3e94468..8879b44 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
@@ -21,7 +21,10 @@ import json
import requests
-### This will make the API calls to activate and confirm an array of users
+__author__ = 'Jeff.West@yahoo.com'
+
+
+# This will make the API calls to activate and confirm an array of users
users = [
'user1@example.com',
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/groups/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/groups/__init__.py b/utils/usergrid-util-python/usergrid_tools/groups/__init__.py
index e69de29..cb3e030 100644
--- a/utils/usergrid-util-python/usergrid_tools/groups/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/groups/__init__.py
@@ -0,0 +1,2 @@
+__author__ = 'Jeff.West@yahoo.com'
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
index 3e88dfe..ba031f3 100644
--- a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
+++ b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
@@ -26,6 +26,9 @@ import urllib3
import requests
+__author__ = 'Jeff.West@yahoo.com'
+
+
group_name = 'precisely-10k'
users = 10000
username_template = 'precisely-10k-%s'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py b/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
index b64a076..3b2a4e0 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
@@ -15,4 +15,7 @@
# * KIND, either express or implied. See the License for the
# * specific language governing permissions and limitations
# * under the License.
-# */
\ No newline at end of file
+# */
+
+__author__ = 'Jeff.West@yahoo.com'
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
index 1ad8fbc..8b1aae6 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
@@ -32,6 +32,9 @@ from logging.handlers import RotatingFileHandler
import sys
+__author__ = 'Jeff.West@yahoo.com'
+
+
entity_template = {
"id": "replaced",
"dataType": "entitlements",
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
index 8ec73f6..d042d38 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
@@ -30,6 +30,9 @@ from logging.handlers import RotatingFileHandler
import sys
+__author__ = 'Jeff.West@yahoo.com'
+
+
entity_template = {
"id": "replaced",
"dataType": "entitlements",
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
index c0f2ebd..ea42c00 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
@@ -25,6 +25,9 @@ from logging.handlers import RotatingFileHandler
import datetime
from usergrid import UsergridQueryIterator
+__author__ = 'Jeff.West@yahoo.com'
+
+
execution_id = str(uuid.uuid4())
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
index e5e860f..08f2bf7 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
@@ -30,7 +30,7 @@ import requests
import traceback
import urllib3.contrib.pyopenssl
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
urllib3.disable_warnings()
urllib3.contrib.pyopenssl.inject_into_urllib3()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
index 632aa68..7929a58 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
@@ -30,7 +30,7 @@ import argparse
from usergrid import UsergridClient, UsergridError
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
logger = logging.getLogger('UsergridIterator')
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/library_check.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/library_check.py b/utils/usergrid-util-python/usergrid_tools/library_check.py
index 0fc6b2f..d053057 100644
--- a/utils/usergrid-util-python/usergrid_tools/library_check.py
+++ b/utils/usergrid-util-python/usergrid_tools/library_check.py
@@ -19,6 +19,9 @@
import traceback
+__author__ = 'Jeff.West@yahoo.com'
+
+
url_data = {
"api_url": "https://usergrid-e2e-prod.e2e.apigee.net/appservices-2-1/",
"org": "",
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/migration/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/__init__.py b/utils/usergrid-util-python/usergrid_tools/migration/__init__.py
index f09d5b5..7247862 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/__init__.py
@@ -1,2 +1,24 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import usergrid_data_migrator
-import usergrid_data_exporter
\ No newline at end of file
+import usergrid_data_exporter
+
+__author__ = 'Jeff.West@yahoo.com'
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
index edfa1c6..0cbb9e1 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
@@ -40,7 +40,7 @@ import signal
from usergrid import UsergridQueryIterator
import urllib3
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
ECID = str(uuid.uuid1())
key_version = 'v4'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
index 42a74ea..ae3f492 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
@@ -42,7 +42,7 @@ from requests.auth import HTTPBasicAuth
from usergrid import UsergridQueryIterator
import urllib3
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
ECID = str(uuid.uuid1())
key_version = 'v4'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
index b64a076..3b2a4e0 100644
--- a/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
@@ -15,4 +15,7 @@
# * KIND, either express or implied. See the License for the
# * specific language governing permissions and limitations
# * under the License.
-# */
\ No newline at end of file
+# */
+
+__author__ = 'Jeff.West@yahoo.com'
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
index 0989401..ed89116 100644
--- a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
+++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
@@ -31,7 +31,7 @@ import traceback
from usergrid import Usergrid
from usergrid.UsergridClient import UsergridEntity
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
logger = logging.getLogger('UsergridParseImporter')
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
index 95730d5..3da722b 100644
--- a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
+++ b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
@@ -22,6 +22,9 @@ from multiprocessing import Pool
import requests
+__author__ = 'Jeff.West@yahoo.com'
+
+
# URL Templates for Usergrid
import time
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py b/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
index e737cb4..2e3abf4 100644
--- a/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
+++ b/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
@@ -29,7 +29,7 @@ import boto
from boto import sqs
import requests
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
sqs_conn = None
sqs_queue = None
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py b/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
index 080c815..139d9a6 100644
--- a/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
+++ b/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
@@ -32,7 +32,7 @@ from multiprocessing import Process, Queue
from boto.sqs.message import RawMessage
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
def total_seconds(td):
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py b/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
index b41894a..1e0983a 100644
--- a/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
+++ b/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
@@ -28,7 +28,7 @@ import boto
from boto import sqs
from multiprocessing import Process, Queue
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
def total_seconds(td):
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py b/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
index 01639e4..6ccdd50 100644
--- a/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
@@ -23,6 +23,9 @@ from collections import defaultdict
import redis
import time
+__author__ = 'Jeff.West@yahoo.com'
+
+
cache = redis.StrictRedis(host='localhost', port=6379, db=0)
# cache.flushall()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac9cfff1/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py b/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
index 0bf3f13..c97a28a 100644
--- a/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
+++ b/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
@@ -19,6 +19,9 @@
import redis
+__author__ = 'Jeff.West@yahoo.com'
+
+
r = redis.Redis("localhost", 6379)
for key in r.scan_iter():
# print '%s: %s' % (r.ttl(key), key)
[18/50] [abbrv] usergrid git commit: 1. added /management/tokendetails?token=&provider=&keyurl 2. POST /management/token -> with grant_type : password and super user —> should create a UG token and return. 3. GET /management/me?access_token= —> with supe
Posted by mr...@apache.org.
1. added /management/tokendetails?token=&provider=&keyurl
2. POST /management/token -> with grant_type : password and super user \u2014> should create a UG token and return.
3. GET /management/me?access_token= \u2014> with super user token -> should return superuser info.
4. Inprogress --> superuser be able to add an org without providing all the administer details.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fad65a8d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fad65a8d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fad65a8d
Branch: refs/heads/master
Commit: fad65a8dff462c0521ceb46585b1bc5f969a926a
Parents: 6e093bc
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Thu Jul 14 00:48:56 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Thu Jul 14 00:48:56 2016 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 6 +-
.../rest/management/ManagementResource.java | 62 ++++++++++++++++++--
.../organizations/OrganizationsResource.java | 11 +++-
.../security/shiro/filters/SecurityFilter.java | 2 +-
.../cassandra/ManagementServiceImpl.java | 6 --
...alSSOProviderAdminUserNotFoundException.java | 17 ++++++
.../security/sso/ApigeeSSO2Provider.java | 41 ++++++++++---
.../security/sso/ExternalSSOProvider.java | 4 ++
.../security/sso/SSOProviderFactory.java | 31 +++++++---
.../security/sso/UsergridExternalProvider.java | 14 ++++-
10 files changed, 159 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index fe739b9..371b251 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -19,7 +19,7 @@
# USERGRID DEPLOYMENT PROPERTIES
###############################################################################
#
-# Contained below are properties used to configure the Usergrid application.
+# Contained below are propertiefs used to configure the Usergrid application.
# Some of the core settings depend on the specific Usergrid deployment architecture.
# For more info, check the docs at:
# <http://usergrid.readthedocs.org/en/two-dot-o-instructions/index.html>
@@ -496,7 +496,6 @@ usergrid.central.read.timeout=10000
usergrid.external.sso.enabled=false
usergrid.external.sso.provider=
usergrid.external.sso.url=
-usergrid.external.sso.publicKeyUrl=
############################### Usergrid Assets #############################
#
@@ -541,6 +540,9 @@ usergrid.sysadmin.login.email=super@usergrid.com
usergrid.sysadmin.login.password=test
usergrid.sysadmin.login.allowed=true
+#enable if superuser can create an org without any user associated with it.
+usergrid.superuser.addorg.enable=true
+
# if usergrid.sysadmin.login.allowed=true, only allows sysadmin login if request is localhost
# if usergrid.sysadmin.login.allowed=false, this property has no effect
usergrid.sysadmin.localhost.only=false
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index c94987a..77569d6 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -39,6 +39,8 @@ import org.apache.usergrid.security.shiro.principals.PrincipalIdentifier;
import org.apache.usergrid.security.shiro.utils.SubjectUtils;
import org.apache.usergrid.security.sso.ExternalSSOProvider;
import org.apache.usergrid.security.sso.SSOProviderFactory;
+import org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl;
+import org.apache.usergrid.utils.JsonUtils;
import org.glassfish.jersey.server.mvc.Viewable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,8 +59,8 @@ import java.util.Map;
import static javax.servlet.http.HttpServletResponse.*;
import static javax.ws.rs.core.MediaType.*;
import static org.apache.commons.lang.StringUtils.isNotBlank;
-import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_SSO_ENABLED;
import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER_URL;
+import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_SSO_ENABLED;
import static org.apache.usergrid.utils.JsonUtils.mapToJsonString;
import static org.apache.usergrid.utils.StringUtils.stringOrSubstringAfterFirst;
import static org.apache.usergrid.utils.StringUtils.stringOrSubstringBeforeFirst;
@@ -191,15 +193,12 @@ public class ManagementResource extends AbstractContextResource {
}
- if(ssoEnabled){
-
+ if(ssoEnabled && !user.getUsername().equals(properties.getProperty(USERGRID_SYSADMIN_LOGIN_NAME))){
ExternalSSOProvider provider = ssoProviderFactory.getProvider();
-
tokenTtl =
Long.valueOf(provider.getDecodedTokenDetails(access_token).get("expiry")) - System.currentTimeMillis()/1000;
}else{
-
tokenTtl = tokens.getTokenInfo(access_token).getDuration();
}
@@ -215,6 +214,51 @@ public class ManagementResource extends AbstractContextResource {
}
+ /**
+ * Get token details. Specially used for external tokens.
+ * @param ui
+ * @param authorization
+ * @param token
+ * @param provider
+ * @param keyUrl
+ * @param callback
+ * @return the json with all the token details. Error message if the external SSO provider is not supported or any other error.
+ * @throws Exception
+ */
+ @GET
+ @Path( "tokendetails" )
+ public Response getTokenDetails( @Context UriInfo ui, @HeaderParam( "Authorization" ) String authorization,
+ @QueryParam( "token" ) String token,
+ @QueryParam( "provider" ) @DefaultValue( "" ) String provider,
+ @QueryParam( "keyurl" ) @DefaultValue( "" ) String keyUrl,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback
+ ) throws Exception {
+
+ ExternalSSOProvider externalprovider = null;
+ Map<String, Object> jwt = null;
+
+ if (! provider.isEmpty()) {
+ //check if its in one of the external provider list.
+ if (!ssoProviderFactory.getProvidersList().contains(StringUtils.upperCase(provider))) {
+ throw new IllegalArgumentException("Unsupported provider.");
+ } else {
+ //get the specific provider.
+ externalprovider = ssoProviderFactory.getSpecificProvider(provider);
+ }
+ }
+ else{ //if the provider is not specified get the default provider enabled in the properties.
+ externalprovider = ssoProviderFactory.getProvider();
+ }
+
+ if(keyUrl.isEmpty()) {
+ keyUrl = externalprovider.getExternalSSOUrl();
+ }
+
+ jwt = externalprovider.getAllTokenDetails(token, keyUrl);
+
+ return Response.status( SC_OK ).type( jsonMediaType( callback ) )
+ .entity( wrapWithCallback(JsonUtils.mapToJsonString(jwt) , callback ) ).build();
+ }
@GET
@Path( "token" )
@@ -336,6 +380,12 @@ public class ManagementResource extends AbstractContextResource {
.entity( wrapWithCallback( response.getBody(), callback ) ).build();
}
+ //moved the check for sso enabled form MangementServiceImpl since was unable to get the current user there to check if its super user.
+ if( tokens.isExternalSSOProviderEnabled() && !user.getUsername().equals(properties.getProperty(USERGRID_SYSADMIN_LOGIN_NAME)) ){
+ throw new RuntimeException("SSO Integration is enabled, Admin users must login via provider: "+
+ properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER));
+ }
+
String token = management.getAccessTokenForAdminUser( user.getUuid(), ttl );
Long passwordChanged = management.getLastAdminPasswordChange( user.getUuid() );
@@ -520,6 +570,8 @@ public class ManagementResource extends AbstractContextResource {
else {
redirect_uri += "&";
}
+
+ //todo: check if sso enabled.
redirect_uri += "code=" + management.getAccessTokenForAdminUser( user.getUuid(), 0 );
if ( isNotBlank( state ) ) {
redirect_uri += "&state=" + URLEncoder.encode( state, "UTF-8" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
index 476e315..eb70486 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.RootResource;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.apache.usergrid.security.shiro.utils.SubjectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -55,6 +56,8 @@ public class OrganizationsResource extends AbstractContextResource {
public static final String ORGANIZATION_PROPERTIES = "properties";
public static final String ORGANIZATION_CONFIGURATION = "configuration";
+ public static final String USERGRID_SYSADMIN_LOGIN_NAME = "usergrid.sysadmin.login.name";
+ public static final String USERGRID_SUPERUSER_ADDORG_ENABLED ="usergrid.superuser.addorg.enable";
@Autowired
private ApplicationCreator applicationCreator;
@@ -185,9 +188,13 @@ public class OrganizationsResource extends AbstractContextResource {
String email, String password, Map<String, Object> userProperties,
Map<String, Object> orgProperties, String callback ) throws Exception {
+ String tokenUserName = SubjectUtils.getUser().getUsername();
+
if ( tokens.isExternalSSOProviderEnabled() ) {
- throw new IllegalArgumentException( "Organization / Admin Users must be created via " +
- properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
+ if(!tokenUserName.equals(properties.getProperty(USERGRID_SYSADMIN_LOGIN_NAME))) {
+ throw new IllegalArgumentException("Organization / Admin Users must be created via " +
+ properties.getProperty(USERGRID_EXTERNAL_PROVIDER_URL));
+ }
}
Preconditions
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java
index 1c06aed..817464f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java
@@ -137,7 +137,7 @@ public abstract class SecurityFilter implements ContainerRequestFilter {
// if this is a CORS Pre-Flight request, we can skip the security check
// OPTIONS requests do not have access into Usergrid data, Jersey default handles these requests
- if( request.getMethod().equalsIgnoreCase("options")){
+ if( request.getMethod().equalsIgnoreCase("options") ){
return true;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index e812b75..90eb4c9 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -68,7 +68,6 @@ import org.apache.usergrid.security.shiro.utils.SubjectUtils;
import org.apache.usergrid.security.tokens.TokenCategory;
import org.apache.usergrid.security.tokens.TokenInfo;
import org.apache.usergrid.security.tokens.TokenService;
-import org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl;
import org.apache.usergrid.security.tokens.exceptions.TokenException;
import org.apache.usergrid.services.*;
import org.apache.usergrid.utils.*;
@@ -1530,11 +1529,6 @@ public class ManagementServiceImpl implements ManagementService {
@Override
public String getAccessTokenForAdminUser( UUID userId, long duration ) throws Exception {
- if( properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_SSO_ENABLED).equalsIgnoreCase("true")){
- throw new RuntimeException("SSO Integration is enabled, Admin users must login via provider: "+
- properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER));
- }
-
return getTokenForPrincipal( ACCESS, null, smf.getManagementAppId(), ADMIN_USER, userId, duration );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java b/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java
index 67f1b1e..dabe9b9 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.usergrid.management.exceptions;
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
index 953da55..61a1601 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
@@ -26,6 +26,7 @@ import org.apache.usergrid.security.AuthPrincipalType;
import org.apache.usergrid.security.tokens.TokenInfo;
import org.apache.usergrid.security.tokens.exceptions.BadTokenException;
import org.apache.usergrid.security.tokens.exceptions.ExpiredTokenException;
+import org.apache.usergrid.utils.JsonUtils;
import org.apache.usergrid.utils.UUIDUtils;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.jackson.JacksonFeature;
@@ -36,7 +37,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import java.security.KeyFactory;
+import java.security.NoSuchAlgorithmException;
import java.security.PublicKey;
+import java.security.spec.InvalidKeySpecException;
import java.security.spec.X509EncodedKeySpec;
import java.util.HashMap;
import java.util.Map;
@@ -64,12 +67,16 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
client = ClientBuilder.newClient(clientConfig);
}
- private String getPublicKey() {
+ public String getPublicKey(String keyUrl) {
- final String keyUrl = properties.getProperty(USERGRID_EXTERNAL_PUBLICKEY_URL);
if(keyUrl != null && !keyUrl.isEmpty()) {
- Map<String, Object> publicKey = client.target(properties.getProperty(USERGRID_EXTERNAL_PUBLICKEY_URL)).request().get(Map.class);
- return publicKey.get(RESPONSE_PUBLICKEY_VALUE).toString().split("----\n")[1].split("\n---")[0];
+ try {
+ Map<String, Object> publicKey = client.target(keyUrl).request().get(Map.class);
+ return publicKey.get(RESPONSE_PUBLICKEY_VALUE).toString().split("----\n")[1].split("\n---")[0];
+ }
+ catch(Exception e){
+ throw new IllegalArgumentException("error getting public key");
+ }
}
return null;
@@ -121,18 +128,28 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
}
+ @Override
+ public Map<String, Object> getAllTokenDetails(String token, String keyUrl) throws Exception {
+ Jws<Claims> claims = getClaimsForKeyUrl(token,getPublicKey(keyUrl));
+ return JsonUtils.toJsonMap(claims.getBody());
- private Jws<Claims> getClaims(String token) throws Exception{
+ }
+ @Override
+ public String getExternalSSOUrl() {
+ return properties.getProperty(USERGRID_EXTERNAL_PUBLICKEY_URL);
+ }
+
+ public Jws<Claims> getClaimsForKeyUrl(String token, String ssoPublicKey) throws NoSuchAlgorithmException, InvalidKeySpecException, BadTokenException {
Jws<Claims> claims = null;
- if(publicKey == null){
+ if(ssoPublicKey == null){
throw new IllegalArgumentException("Public key must be provided with Apigee " +
"token in order to verify signature.");
}
- byte[] publicBytes = decodeBase64(publicKey);
+ byte[] publicBytes = decodeBase64(ssoPublicKey);
X509EncodedKeySpec keySpec = new X509EncodedKeySpec(publicBytes);
KeyFactory keyFactory = KeyFactory.getInstance("RSA");
PublicKey pubKey = keyFactory.generatePublic(keySpec);
@@ -141,7 +158,7 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
claims = Jwts.parser().setSigningKey(pubKey).parseClaimsJws(token);
} catch (SignatureException se) {
if(logger.isDebugEnabled()) {
- logger.debug("Signature was invalid for Apigee JWT: {} and key: {}", token, publicKey);
+ logger.debug("Signature was invalid for Apigee JWT: {} and key: {}", token, ssoPublicKey);
}
throw new BadTokenException("Invalid Apigee SSO token signature");
} catch (MalformedJwtException me){
@@ -160,6 +177,12 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
return claims;
}
+ public Jws<Claims> getClaims(String token) throws Exception{
+
+ return getClaimsForKeyUrl(token,publicKey);
+
+ }
+
private void validateClaims (final Jws<Claims> claims) throws ExpiredTokenException {
final Claims body = claims.getBody();
@@ -185,6 +208,6 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
@Autowired
public void setProperties(Properties properties) {
this.properties = properties;
- this.publicKey = getPublicKey();
+ this.publicKey = getPublicKey(getExternalSSOUrl());
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java
index 180a675..ebd7ec5 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java
@@ -35,4 +35,8 @@ public interface ExternalSSOProvider {
/** Decode the token, if supported, and return any information encoded with the token */
Map<String, String> getDecodedTokenDetails(String token) throws Exception;
+ Map<String, Object> getAllTokenDetails(String token, String keyUrl) throws Exception;
+
+ String getExternalSSOUrl() throws Exception;
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java
index a3016c8..2b9755e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java
@@ -21,7 +21,10 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl;
import org.springframework.beans.factory.annotation.Autowired;
+import java.util.List;
import java.util.Properties;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
/**
* Created by russo on 6/24/16.
@@ -38,17 +41,21 @@ public class SSOProviderFactory {
public ExternalSSOProvider getProvider(){
- final Provider configuredProvider;
+ return getSpecificProvider(properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER));
+
+ }
+
+ public ExternalSSOProvider getSpecificProvider(String providerName){
+
+ final Provider specifiedProvider ;
try{
- configuredProvider =
- Provider.valueOf(properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER).toUpperCase());
- }catch (IllegalArgumentException e){
- throw new RuntimeException("Property usergrid.external.sso.provider must " +
- "be configured when external SSO is enabled");
+ specifiedProvider = Provider.valueOf(providerName.toUpperCase());
+ }
+ catch(IllegalArgumentException e){
+ throw new IllegalArgumentException("Unsupported provider");
}
- switch (configuredProvider){
-
+ switch (specifiedProvider){
case APIGEE:
return ((CpEntityManagerFactory)emf).getApplicationContext().getBean( ApigeeSSO2Provider.class );
case USERGRID:
@@ -56,9 +63,9 @@ public class SSOProviderFactory {
default:
throw new RuntimeException("Unknown SSO provider");
}
-
}
+
@Autowired
public void setEntityManagerFactory( EntityManagerFactory emf ) {
this.emf = emf;
@@ -69,4 +76,10 @@ public class SSOProviderFactory {
public void setProperties(Properties properties) {
this.properties = properties;
}
+
+ public List<String> getProvidersList() {
+ return Stream.of(Provider.values())
+ .map(Enum::name)
+ .collect(Collectors.toList());
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fad65a8d/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java
index a2e5fb2..0cfe2d0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java
@@ -48,7 +48,7 @@ public class UsergridExternalProvider implements ExternalSSOProvider {
private static final String SSO_PROCESSING_TIME = "sso.processing_time";
private static final String SSO_TOKENS_REJECTED = "sso.tokens_rejected";
private static final String SSO_TOKENS_VALIDATED = "sso.tokens_validated";
- public static final String USERGRID_CENTRAL_URL = "usergrid.external.sso.publicKeyUrl";
+ public static final String USERGRID_CENTRAL_URL = "usergrid.external.sso.url";
public static final String CENTRAL_CONNECTION_POOL_SIZE = "usergrid.central.connection.pool.size";
public static final String CENTRAL_CONNECTION_TIMEOUT = "usergrid.central.connection.timeout";
public static final String CENTRAL_READ_TIMEOUT = "usergrid.central.read.timeout";
@@ -91,6 +91,18 @@ public class UsergridExternalProvider implements ExternalSSOProvider {
}
@Override
+ public Map<String, Object> getAllTokenDetails(String token, String keyUrl) throws Exception {
+ throw new UnsupportedOperationException("Returning all token details info not supported from external Usergrid SSO tokens");
+
+ }
+
+ @Override
+ public String getExternalSSOUrl() {
+ return properties.getProperty(USERGRID_CENTRAL_URL);
+ }
+
+
+ @Override
public UserInfo validateAndReturnUserInfo(String token, long ttl) throws Exception {
if (token == null) {
throw new IllegalArgumentException("ext_access_token must be specified");
[27/50] [abbrv] usergrid git commit: Initial checkin for Python
Utilities and SDK
Posted by mr...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
new file mode 100644
index 0000000..98d4373
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
@@ -0,0 +1,270 @@
+import json
+import re
+import traceback
+from multiprocessing.pool import Pool
+import requests
+
+index_url_template = 'http://localhost:9200/{index_name}/_search?size={size}&from={from_var}&q=-edgeName:zzzcollzzz|logs'
+
+index_names = [
+ 'es-index-name'
+]
+
+baas_url = 'http://localhost:8080/org/{app_id}/{collection}/{entity_id}'
+
+field_part_map = {
+ 'mockdata': 'mockData'
+}
+
+
+def update_entity_field(entity, field_name, field_value):
+ entity_copy = entity.copy()
+
+ worked = True
+ is_array = False
+ array_length = 0
+
+ try:
+ parts = field_name.split('.')
+
+ if parts[len(parts) - 1] != 'size':
+ print parts
+ exit()
+
+ change_me = entity_copy
+
+ for i, field_part in enumerate(parts):
+ field_part = field_part_map.get(field_part, field_part)
+
+ if field_part == 'size':
+ break
+
+ if isinstance(change_me, dict):
+ if field_part not in change_me:
+ worked = False
+ # print 'ERROR! field [%s] not in entity: %s' % (field_part, json.dumps(change_me))
+ break
+
+ change_me = change_me[field_part]
+
+ elif isinstance(change_me, list):
+ array_length = len(change_me)
+
+ if i == len(parts) - 2 and len(parts) > i + 1 and parts[i + 1] == 'size':
+
+ for j in xrange(0, len(change_me)):
+ print 'arrau!'
+ change_me[j] = update_entity_field(change_me[j], '.'.join(parts[i:]), field_value)
+ # element['size'] = field_value
+
+ elif len(change_me) == 1:
+ print 'single array'
+ change_me = change_me[0][field_part]
+ else:
+ print 'WTF!'
+ try:
+ change_me['size'] = field_value
+ except:
+ if array_length != 1:
+ print traceback.format_exc()
+ print 'damn'
+
+ except:
+ print '---Error updating field [%s] in document: %s' % (field_name, json.dumps(entity))
+ print traceback.format_exc()
+
+ if array_length > 1:
+ print '++++++++ARRAY!!!!! %s' % array_length
+
+ return entity_copy
+
+
+def update_entity_fields(entity, fields):
+ entity_copy = entity.copy()
+
+ for field in fields:
+ field_name = field.get('name')
+
+ if 'string' in field:
+ field_value = field.get('string')
+
+ elif 'long' in field:
+ field_value = field.get('long')
+
+ else:
+ print 'WTF! %s' % json.dumps(field)
+ return entity_copy
+
+ entity_copy = update_entity_field(entity_copy, field_name, field_value)
+
+ return entity_copy
+
+
+my = {
+ 'foo': {
+ 'bar': {
+ 'color': 'red'
+ }
+ }
+}
+
+fields = [
+ {
+ 'name': 'foo.size',
+ 'string': '2'
+ },
+ {
+ 'name': 'foo.bar.size',
+ 'long': 2
+ }
+]
+
+
+def work(item):
+ try:
+ url = 'http://localhost:8080/org/{app_id}/{collection}/{entity_id}'.format(
+ app_id=item[0],
+ collection=item[1],
+ entity_id=item[2]
+ )
+ print url
+ r_get = requests.get(url)
+
+ if r_get.status_code != 200:
+ print 'ERROR GETTING ENTITY AT URL: %s' % url
+ return
+
+ response_json = r_get.json()
+
+ entities = response_json.get('entities')
+
+ if len(entities) <= 0:
+ print 'TOO MANY ENTITIES AT URL: %s' % url
+ return
+
+ entity = entities[0]
+
+ new_entity = update_entity_fields(entity, item[3])
+
+ with open('/Users/ApigeeCorporation/tmp/hack/%s.json' % item[2], 'w') as f:
+ json.dump(entity, f, indent=2)
+
+ with open('/Users/ApigeeCorporation/tmp/hack/%s_new.json' % item[2], 'w') as f:
+ json.dump(new_entity, f, indent=2)
+
+ r_put = requests.put(url, data=json.dumps(new_entity))
+
+ if r_put.status_code == 200:
+ print 'PUT [%s]: %s' % (r_put.status_code, url)
+ pass
+ elif r_put.status_code:
+ print 'PUT [%s]: %s | %s' % (r_put.status_code, url, r_put.text)
+
+ except:
+ print traceback.format_exc()
+
+
+POOL_SIZE = 4
+
+counter = 0
+size = POOL_SIZE * 10
+size = 1000
+
+total_docs = 167501577
+start_from = 0
+from_var = 0
+page = 0
+
+work_items = []
+
+pool = Pool(POOL_SIZE)
+
+keep_going = True
+
+while keep_going:
+ work_items = []
+
+ if from_var > total_docs:
+ keep_going = False
+ break
+
+ from_var = start_from + (page * size)
+ page += 1
+
+ for index_name in index_names:
+
+ index_url = index_url_template.format(index_name=index_name, size=size, from_var=from_var)
+
+ print 'Getting URL: ' + index_url
+
+ r = requests.get(index_url)
+
+ if r.status_code != 200:
+ print r.text
+ exit()
+
+ response = r.json()
+
+ hits = response.get('hits', {}).get('hits')
+
+ re_app_id = re.compile('appId\((.+),')
+ re_ent_id = re.compile('entityId\((.+),')
+ re_type = re.compile('entityId\(.+,(.+)\)')
+
+ print 'Index: %s | hits: %s' % (index_name, len(hits))
+
+ if len(hits) == 0:
+ keep_going = False
+ break
+
+ for hit_data in hits:
+ source = hit_data.get('_source')
+
+ application_id = source.get('applicationId')
+
+ app_id_find = re_app_id.findall(application_id)
+
+ if len(app_id_find) > 0:
+ app_id = app_id_find[0]
+
+ if app_id != '5f20f423-f2a8-11e4-a478-12a5923b55dc':
+ print 'SKIPP APP ID: ' + app_id
+ continue
+
+ entity_id_tmp = source.get('entityId')
+
+ entity_id_find = re_ent_id.findall(entity_id_tmp)
+ entity_type_find = re_type.findall(entity_id_tmp)
+
+ if len(entity_id_find) > 0 and len(entity_type_find) > 0:
+ entity_id = entity_id_find[0]
+ collection = entity_type_find[0]
+ fields_to_update = []
+
+ for field in source.get('fields'):
+ if field.get('name')[-5:] == '.size':
+ fields_to_update.append(field)
+
+ print json.dumps(source)
+
+ work_items.append((app_id, collection, entity_id, fields_to_update))
+
+ counter += 1
+
+ print 'Work Items: %s' % len(work_items)
+
+ try:
+ pool.map(work, work_items)
+
+
+ except:
+ print traceback.format_exc()
+
+ try:
+ pool.map(work, work_items)
+ except:
+ pass
+
+ print 'Work Done!'
+
+print 'done: %s' % counter
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/index_prefix_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_prefix_checker.py b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
new file mode 100644
index 0000000..d72ff3d
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
@@ -0,0 +1,81 @@
+import json
+from collections import defaultdict
+import requests
+import logging
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+# This script iterates all the indexes in an ES cluster and aggregates the size by the prefix
+
+url_base = 'http://localhost:9200'
+
+r = requests.get(url_base + "/_stats")
+response = r.json()
+
+indices = r.json()['indices']
+
+print 'retrieved %s indices' % len(indices)
+
+NUMBER_VALUE = 0
+
+includes = [
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
+]
+
+excludes = [
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c8',
+]
+
+counter = 0
+process = False
+
+counts = defaultdict(int)
+sizes = defaultdict(int)
+indexes = {}
+
+for index, index_data in indices.iteritems():
+ process = False
+ counter += 1
+
+ if 'management' in index:
+ print index
+
+ # print 'index %s of %s' % (counter, len(indices))
+
+ if len(includes) == 0:
+ process = True
+ else:
+ for include in includes:
+
+ if include in index:
+ process = True
+
+ if len(excludes) > 0:
+ for exclude in excludes:
+ if exclude in index:
+ process = False
+
+ if process:
+ # print index
+ if '__' in index:
+ index_prefix = index.split('__')[0]
+ elif '^' in index:
+ index_prefix = index.split('^')[0]
+ else:
+ index_prefix = index.split('_')[0]
+
+ if index_prefix not in indexes:
+ indexes[index_prefix] = []
+
+ indexes[index_prefix].append(index)
+
+ counts[index_prefix] += 1
+ counts['total'] += 1
+ sizes[index_prefix] += (float(index_data.get('total', {}).get('store', {}).get('size_in_bytes')) / 1e+9)
+ sizes['total'] += (float(index_data.get('total', {}).get('store', {}).get('size_in_bytes')) / 1e+9)
+
+print 'Number of indices (US-EAST):'
+print json.dumps(counts, indent=2)
+print 'Size in GB'
+print json.dumps(sizes, indent=2)
+print json.dumps(indexes, indent=2)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/index_replica_setter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_replica_setter.py b/utils/usergrid-util-python/es_tools/index_replica_setter.py
new file mode 100644
index 0000000..7180fed
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/index_replica_setter.py
@@ -0,0 +1,118 @@
+from multiprocessing import Pool
+import requests
+import time
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+# utility for updating the replicas of a set of indexes that are no longer needed. Given:
+# A) a set of strings to include when evaluating the index names to update
+# B) a set of strings to Exclude when evaluating the index names to update
+#
+# The general logic is:
+# 1) If the include set is empty, or if the index name contains a string in the 'include' set, then update
+# 2) If the index contains a string in the exclude list, do not update
+
+
+url_base = 'http://localhost:9200'
+
+# r = requests.get(url_base + "/_cat/indices?v")
+# print r.text
+
+r = requests.get(url_base + "/_stats")
+
+# print json.dumps(r.json(), indent=2)
+
+indices = r.json()['indices']
+
+print 'retrieved %s indices' % len(indices)
+
+NUMBER_VALUE = 1
+
+payload = {
+ "index.number_of_replicas": NUMBER_VALUE,
+}
+
+# indices = ['usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_manual']
+
+includes = [
+ # '70be096e-c2e1-11e4-8a55-12b4f5e28868',
+ # 'b0c640af-bc6c-11e4-b078-12b4f5e28868',
+ # 'e62e465e-bccc-11e4-b078-12b4f5e28868',
+ # 'd82b6413-bccc-11e4-b078-12b4f5e28868',
+ # '45914256-c27f-11e4-8a55-12b4f5e28868',
+ # '2776a776-c27f-11e4-8a55-12b4f5e28868',
+ # 'a54f878c-bc6c-11e4-b044-0e4cd56e19cd',
+ # 'ed5b47ea-bccc-11e4-b078-12b4f5e28868',
+ # 'bd4874ab-bccc-11e4-b044-0e4cd56e19cd',
+ # '3d748996-c27f-11e4-8a55-12b4f5e28868',
+ # '1daab807-c27f-11e4-8a55-12b4f5e28868',
+ # 'd0c4f0da-d961-11e4-849d-12b4f5e28868',
+ # '93e756ac-bc4e-11e4-92ae-12b4f5e28868',
+]
+
+excludes = [
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c8',
+ # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3',
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
+ # 'a34ad389-b626-11e4-848f-06b49118d7d0'
+]
+
+counter = 0
+update = False
+# print 'sleeping 1200s'
+# time.sleep(1200)
+
+index_names = sorted([index for index in indices])
+
+
+def update_shards(index_name):
+ update = False
+ # counter += 1
+ #
+ # print 'index %s of %s' % (counter, len(indices))
+
+ if len(includes) == 0:
+ update = True
+ else:
+ for include in includes:
+
+ if include in index_name:
+ update = True
+
+ if len(excludes) > 0:
+ for exclude in excludes:
+ if exclude in index_name:
+ update = False
+
+ if update:
+ print index_name
+
+ # url = '%s/%s/_settings' % (url_base, index)
+ # print url
+ #
+ # response = requests.get('%s/%s/_settings' % (url_base, index))
+ # settings = response.json()
+ #
+ # index_settings = settings[index]['settings']['index']
+ #
+ # current_replicas = int(index_settings.get('number_of_replicas'))
+ #
+ # if current_replicas == NUMBER_VALUE:
+ # continue
+
+ success = False
+
+ while not success:
+
+ response = requests.put('%s/%s/_settings' % (url_base, index_name), data=payload)
+
+ if response.status_code == 200:
+ success = True
+ print '200: %s: %s' % (index_name, response.text)
+ else:
+ print '%s: %s: %s' % (response.status_code, index_name, response.text)
+
+
+pool = Pool(8)
+
+pool.map(update_shards, index_names)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/index_shard_allocator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_shard_allocator.py b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
new file mode 100644
index 0000000..ecee095
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
@@ -0,0 +1,148 @@
+import json
+from multiprocessing import Pool
+
+import requests
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+# The purpose of this script is to update the shard allocation of ElasticSearch for specific indexes to be set to
+# specific nodes. The reason for doing this is to isolate the nodes on which certain indexes run for specific
+# customers due to load, disk size or any other factors.
+
+
+nodes_c32xl = [
+ 'res000eu',
+ 'res001eu',
+ 'res002eu',
+ 'res003eu',
+ 'res004eu',
+ 'res005eu',
+ 'res009eu',
+ 'res010eu',
+ 'res011eu',
+ 'res012eu',
+ 'res013eu',
+ 'res014eu',
+]
+
+nodes_c34xl = [
+ 'res015eu',
+ 'res018eu',
+ 'res019eu',
+ 'res020eu',
+ 'res021eu',
+ 'res022eu',
+ 'res023eu',
+ 'res024eu',
+ 'res025eu',
+ 'res026eu',
+ 'res027eu',
+ 'res028eu'
+]
+
+nodes = nodes_c34xl
+
+url_base = 'http://localhost:9200'
+
+nodes_string = ",".join(nodes)
+
+payload = {
+ "index.routing.allocation.include._host": "",
+ "index.routing.allocation.exclude._host": nodes_string
+}
+
+# payload = {
+# "index.routing.allocation.include._host": "",
+# "index.routing.allocation.exclude._host": ""
+# }
+
+print json.dumps(payload )
+
+
+r = requests.get(url_base + "/_stats")
+indices = r.json()['indices']
+
+print 'retrieved %s indices' % len(indices)
+
+includes = [
+ # '70be096e-c2e1-11e4-8a55-12b4f5e28868',
+ # 'b0c640af-bc6c-11e4-b078-12b4f5e28868',
+ # 'e62e465e-bccc-11e4-b078-12b4f5e28868',
+ # 'd82b6413-bccc-11e4-b078-12b4f5e28868',
+ # '45914256-c27f-11e4-8a55-12b4f5e28868',
+ # '2776a776-c27f-11e4-8a55-12b4f5e28868',
+ # 'a54f878c-bc6c-11e4-b044-0e4cd56e19cd',
+ # 'ed5b47ea-bccc-11e4-b078-12b4f5e28868',
+ # 'bd4874ab-bccc-11e4-b044-0e4cd56e19cd',
+ # '3d748996-c27f-11e4-8a55-12b4f5e28868',
+ # '1daab807-c27f-11e4-8a55-12b4f5e28868',
+ # 'd0c4f0da-d961-11e4-849d-12b4f5e28868',
+ # '93e756ac-bc4e-11e4-92ae-12b4f5e28868',
+ #
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c8',
+ # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3',
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
+]
+
+excludes = [
+ #
+ # '70be096e-c2e1-11e4-8a55-12b4f5e28868',
+ # 'b0c640af-bc6c-11e4-b078-12b4f5e28868',
+ # 'e62e465e-bccc-11e4-b078-12b4f5e28868',
+ # 'd82b6413-bccc-11e4-b078-12b4f5e28868',
+ # '45914256-c27f-11e4-8a55-12b4f5e28868',
+ # '2776a776-c27f-11e4-8a55-12b4f5e28868',
+ # 'a54f878c-bc6c-11e4-b044-0e4cd56e19cd',
+ # 'ed5b47ea-bccc-11e4-b078-12b4f5e28868',
+ # 'bd4874ab-bccc-11e4-b044-0e4cd56e19cd',
+ # '3d748996-c27f-11e4-8a55-12b4f5e28868',
+ # '1daab807-c27f-11e4-8a55-12b4f5e28868',
+ # 'd0c4f0da-d961-11e4-849d-12b4f5e28868',
+ # '93e756ac-bc4e-11e4-92ae-12b4f5e28868',
+ #
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c8',
+ # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3',
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
+]
+
+counter = 0
+update = False
+
+for index_name in indices:
+ update = False
+ counter += 1
+
+ # print 'Checking index %s of %s: %s' % (counter, len(indices), index_name)
+
+ if len(includes) == 0:
+ update = True
+ else:
+ for include in includes:
+
+ if include in index_name:
+ update = True
+
+ if len(excludes) > 0:
+ for exclude in excludes:
+ if exclude in index_name:
+ update = False
+
+ if not update:
+ print 'Skipping %s of %s: %s' % (counter, len(indices), index_name)
+ else:
+ print '+++++Processing %s of %s: %s' % (counter, len(indices), index_name)
+
+ url_template = '%s/%s/_settings' % (url_base, index_name)
+ print url_template
+
+ success = False
+
+ while not success:
+
+ response = requests.put('%s/%s/_settings' % (url_base, index_name), data=json.dumps(payload))
+
+ if response.status_code == 200:
+ success = True
+ print '200: %s: %s' % (index_name, response.text)
+ else:
+ print '%s: %s: %s' % (response.status_code, index_name, response.text)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/mapping_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_deleter.py b/utils/usergrid-util-python/es_tools/mapping_deleter.py
new file mode 100644
index 0000000..74ad898
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/mapping_deleter.py
@@ -0,0 +1,34 @@
+import json
+
+import requests
+
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+url_base = 'http://localhost:9200'
+
+SOURCE_INDEX = '5f20f423-f2a8-11e4-a478-12a5923b55dc__application_v6'
+
+url_template = '%s/{index_name}/_mapping' % url_base
+
+source_index_url = '%s/%s' % (url_base, SOURCE_INDEX)
+
+index_name = SOURCE_INDEX
+
+index_data = requests.get(url_template.format(index_name=index_name)).json()
+
+mappings = index_data.get(index_name, {}).get('mappings', {})
+
+for type_name, mapping_detail in mappings.iteritems():
+ print 'Index: %s | Type: %s: | Properties: %s' % (index_name, type_name, len(mappings[type_name]['properties']))
+
+ if type_name == '_default_':
+ continue
+
+ r = requests.delete('%s/%s/_mapping/%s' % (url_base, index_name, type_name))
+
+ print '%s: %s' % (r.status_code, r.text)
+
+ # print json.dumps(r.json(), indent=2)
+ # time.sleep(5)
+ print '---'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/mapping_retriever.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_retriever.py b/utils/usergrid-util-python/es_tools/mapping_retriever.py
new file mode 100644
index 0000000..0da123b
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/mapping_retriever.py
@@ -0,0 +1,45 @@
+import json
+import requests
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+# Utility to iterate the mappings for an index and save them locally
+
+url_base = 'http://localhost:9200'
+
+# r = requests.get(url_base + "/_stats")
+#
+# indices = r.json()['indices']
+
+url_template = '%s/{index_name}/_mapping' % url_base
+
+SOURCE_INDEX = '5f20f423-f2a8-11e4-a478-12a5923b55dc__application_v7'
+
+source_index_url = '%s/%s' % (url_base, SOURCE_INDEX)
+
+index_name = SOURCE_INDEX
+print 'Getting ' + url_template.format(index_name=index_name)
+
+r = requests.get(url_template.format(index_name=index_name))
+index_data = r.json()
+
+mappings = index_data.get(index_name, {}).get('mappings', {})
+
+for type_name, mapping_detail in mappings.iteritems():
+ print 'Index: %s | Type: %s: | Properties: %s' % (index_name, type_name, len(mappings[type_name]['properties']))
+
+ print 'Processing %s' % type_name
+
+ filename = '/Users/ApigeeCorporation/tmp/%s_%s_source_mapping.json' % (
+ SOURCE_INDEX, type_name)
+
+ print filename
+
+ with open(filename, 'w') as f:
+ json.dump({type_name: mapping_detail}, f, indent=2)
+
+ # print '%s' % (r.status_code, r.text)
+
+ # print json.dumps(r.json(), indent=2)
+ # time.sleep(5)
+ print 'Done!'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/monitor_tasks.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/monitor_tasks.py b/utils/usergrid-util-python/es_tools/monitor_tasks.py
new file mode 100644
index 0000000..df23d49
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/monitor_tasks.py
@@ -0,0 +1,41 @@
+import datetime
+import requests
+import time
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+# Utility for monitoring pending tasks in ElasticSearch
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+url_template = "http://localhost:9200/_cat/pending_tasks?v'"
+
+x = 0
+
+SLEEP_TIME = 3
+
+while True:
+ x += 13
+ try:
+
+ r = requests.get(url=url_template)
+ lines = r.text.split('\n')
+
+ print '\n-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-'
+ print '+++++++++++++++++++++++++++++++++++++++++++++++++++++++++'
+ print datetime.datetime.utcnow()
+ if len(lines) > 1:
+ print r.text
+ else:
+ print 'None'
+
+ print '-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-'
+ print '-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n'
+
+ except:
+ pass
+
+ time.sleep(SLEEP_TIME)
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/index_test/README.md
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/README.md b/utils/usergrid-util-python/index_test/README.md
new file mode 100644
index 0000000..eed7f1c
--- /dev/null
+++ b/utils/usergrid-util-python/index_test/README.md
@@ -0,0 +1 @@
+This set of scripts was intended to test indexing times and sizes for the new indexing scheme
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/index_test/document_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/document_creator.py b/utils/usergrid-util-python/index_test/document_creator.py
new file mode 100644
index 0000000..fd544c6
--- /dev/null
+++ b/utils/usergrid-util-python/index_test/document_creator.py
@@ -0,0 +1,254 @@
+from __future__ import print_function
+from Queue import Empty
+import json
+from multiprocessing import JoinableQueue, Process
+import random
+import re
+import uuid
+import sys
+
+import argparse
+
+import loremipsum
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='ElasticSearch Index Test 1')
+
+ parser.add_argument('-w', '--workers',
+ help='The number of worker threads',
+ type=int,
+ default=8)
+
+ parser.add_argument('-dc', '--document_count',
+ help='The number of documents per index',
+ type=long,
+ default=100000000)
+
+ parser.add_argument('--output',
+ help='The filename to write to',
+ type=str,
+ default='generated_documents.txt')
+
+ parser.add_argument('--fields_min',
+ help='The min number of fields per document',
+ type=long,
+ default=10)
+
+ parser.add_argument('--fields_max',
+ help='The max number of fields per document',
+ type=long,
+ default=100)
+
+ parser.add_argument('-tp', '--type_prefix',
+ help='The Prefix to use for type names',
+ type=str,
+ default='type_this')
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+args = parse_args()
+
+sentence_list = loremipsum.get_sentences(10000)
+
+
+class Worker(Process):
+ def __init__(self, work_queue, response_queue):
+ super(Worker, self).__init__()
+ self.work_queue = work_queue
+ self.response_queue = response_queue
+ self.sentence_list = loremipsum.get_sentences(1000)
+ self.re_first_word = re.compile('([A-z]+)')
+
+ def run(self):
+ print('Starting %s ' % self.name)
+
+ while True:
+ task = self.work_queue.get(timeout=600)
+ field_count = random.randint(task['fields_min'], task['fields_max'])
+ document = self.generate_document(field_count)
+ flattened_doc = self.process_document(document,
+ task['uuid'],
+ task['uuid'])
+
+ self.response_queue.put(flattened_doc)
+
+ self.work_queue.task_done()
+
+ def generate_document(self, fields):
+
+ doc = {}
+
+ my_bool = True
+
+ for i in xrange(fields):
+ sentence_index = random.randint(0, max((fields / 2) - 1, 1))
+ sentence = self.sentence_list[sentence_index]
+
+ if random.random() >= .5:
+ key = self.re_first_word.findall(sentence)[1]
+ else:
+ key = self.re_first_word.findall(sentence)[1] + str(i)
+
+ field_type = random.random()
+
+ if field_type <= 0.3:
+ doc[key] = sentence
+
+ elif field_type <= 0.5:
+ doc[key] = random.randint(1, 1000000)
+
+ elif field_type <= 0.6:
+ doc[key] = random.random() * 1000000000
+
+ elif field_type == 0.7:
+ doc[key] = my_bool
+ my_bool = not my_bool
+
+ elif field_type == 0.8:
+ doc[key] = self.generate_document(max(fields / 5, 1))
+
+ elif field_type <= 1.0:
+ doc['mylocation'] = self.generate_location()
+
+ return doc
+
+ @staticmethod
+ def get_fields(document, base_name=None):
+ fields = []
+
+ for name, value in document.iteritems():
+ if base_name:
+ field_name = '%s.%s' % (base_name, name)
+ else:
+ field_name = name
+
+ if isinstance(value, dict):
+ fields += Worker.get_fields(value, field_name)
+ else:
+ value_name = None
+ if isinstance(value, basestring):
+ value_name = 'string'
+
+ elif isinstance(value, bool):
+ value_name = 'boolean'
+
+ elif isinstance(value, (int, long)):
+ value_name = 'long'
+
+ elif isinstance(value, float):
+ value_name = 'double'
+
+ if value_name:
+ field = {
+ 'name': field_name,
+ value_name: value
+ }
+ else:
+ field = {
+ 'name': field_name,
+ 'string': str(value)
+ }
+
+ fields.append(field)
+
+ return fields
+
+
+ @staticmethod
+ def process_document(document, application_id, uuid):
+ response = {
+ 'entityId': uuid,
+ 'entityVersion': '1',
+ 'applicationId': application_id,
+ 'fields': Worker.get_fields(document)
+ }
+
+ return response
+
+ def generate_location(self):
+ response = {}
+
+ lat = random.random() * 90.0
+ lon = random.random() * 180.0
+
+ lat_neg_true = True if lon > .5 else False
+ lon_neg_true = True if lat > .5 else False
+
+ lat = lat * -1.0 if lat_neg_true else lat
+ lon = lon * -1.0 if lon_neg_true else lon
+
+ response['location'] = {
+ 'lat': lat,
+ 'lon': lon
+ }
+
+ return response
+
+
+class Writer(Process):
+ def __init__(self, document_queue):
+ super(Writer, self).__init__()
+ self.document_queue = document_queue
+
+ def run(self):
+ keep_going = True
+
+ with open(args['output'], 'w') as f:
+ while keep_going:
+ try:
+ document = self.document_queue.get(timeout=300)
+ print(json.dumps(document), file=f)
+
+ except Empty:
+ print('done!')
+ keep_going = False
+
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+def main():
+ work_queue = JoinableQueue()
+ response_queue = JoinableQueue()
+
+ workers = [Worker(work_queue, response_queue) for x in xrange(args.get('workers'))]
+
+ writer = Writer(response_queue)
+ writer.start()
+
+ [worker.start() for worker in workers]
+
+ try:
+ total_messages = args.get('document_count')
+ batch_size = 100000
+ message_counter = 0
+
+ for doc_number in xrange(total_messages):
+ message_counter += 1
+
+ for count in xrange(batch_size):
+ doc_id = str(uuid.uuid1())
+
+ task = {
+ 'fields_min': args['fields_min'],
+ 'fields_max': args['fields_max'],
+ 'uuid': doc_id
+ }
+
+ work_queue.put(task)
+
+ print('Joining queues counter=[%s]...' % message_counter)
+ work_queue.join()
+ response_queue.join()
+ print('Done queue counter=[%s]...' % message_counter)
+
+ except KeyboardInterrupt:
+ [worker.terminate() for worker in workers]
+
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
new file mode 100644
index 0000000..d1dd40c
--- /dev/null
+++ b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
@@ -0,0 +1,545 @@
+import json
+from multiprocessing import JoinableQueue, Process
+import random
+import re
+import traceback
+import uuid
+import time
+import sys
+
+import argparse
+import loremipsum
+import requests
+from elasticsearch import Elasticsearch
+
+
+es_hosts = [
+ {'host': 'ees000wo', 'port': 9200},
+ {'host': 'ees001wo', 'port': 9200},
+ {'host': 'ees002wo', 'port': 9200},
+ {'host': 'ees003wo', 'port': 9200},
+ {'host': 'ees004wo', 'port': 9200},
+ {'host': 'ees005wo', 'port': 9200},
+ {'host': 'ees006wo', 'port': 9200},
+ {'host': 'ees007wo', 'port': 9200},
+ {'host': 'ees008wo', 'port': 9200},
+ {'host': 'ees009wo', 'port': 9200},
+ {'host': 'ees010wo', 'port': 9200},
+ {'host': 'ees011wo', 'port': 9200},
+ {'host': 'ees012wo', 'port': 9200},
+ {'host': 'ees013wo', 'port': 9200},
+ {'host': 'ees014wo', 'port': 9200},
+ {'host': 'ees015wo', 'port': 9200},
+ {'host': 'ees016wo', 'port': 9200},
+ {'host': 'ees017wo', 'port': 9200}
+]
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='ElasticSearch Index Test 1')
+
+ parser.add_argument('-t', '--type_count',
+ help='The number of types to produce',
+ type=int,
+ default=100)
+
+ parser.add_argument('-ic', '--index_count',
+ help='The number of indices to create',
+ type=int,
+ default=10)
+
+ parser.add_argument('-sc', '--shard_count',
+ help='The number of indices to create',
+ type=int,
+ default=18)
+
+ parser.add_argument('-rc', '--replica_count',
+ help='The number of indices to create',
+ type=int,
+ default=1)
+
+ parser.add_argument('-w', '--workers',
+ help='The number of worker threads',
+ type=int,
+ default=8)
+
+ parser.add_argument('-dc', '--document_count',
+ help='The number of documents per index',
+ type=long,
+ default=100000000)
+
+ parser.add_argument('-bs', '--batch_size',
+ help='The size of batches to send to ES',
+ type=long,
+ default=25)
+
+ parser.add_argument('-ip', '--index_prefix',
+ help='The Prefix to use for index names',
+ type=str,
+ default='apigee_ftw')
+
+ parser.add_argument('-tp', '--type_prefix',
+ help='The Prefix to use for type names',
+ type=str,
+ default='type_this')
+
+ parser.add_argument('-s', '--setup',
+ help='The Prefix to use for type names',
+ action='store_true')
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+args = parse_args()
+
+
+class APIClient():
+ def __init__(self, base_url):
+ self.base_url = base_url
+
+ def put(self, path='/', data=None):
+ if not data:
+ data = {}
+
+ url = '%s%s' % (self.base_url, path)
+ r = requests.put(url, json.dumps(data))
+
+ if r.status_code == 200:
+ print 'PUT (%s) in %sms' % (r.status_code, total_milliseconds(r.elapsed))
+ return r.json()
+
+ raise Exception('HTTP %s calling PUT on URL=[%s]: %s' % (r.status_code, url, r.text))
+
+ def index_batch(self, batch):
+
+ data = ''
+
+ for element in batch:
+ index_tuple = element[0]
+ doc = element[1]
+ data += '{ "index" : { "_index" : "%s", "_type" : "%s", "_id" : "%s" } }\n' % (
+ index_tuple[0], index_tuple[1], doc['entityId'])
+ data += json.dumps(doc)
+ data += '\n'
+
+ url = '%s/_bulk' % self.base_url
+
+ # print data
+
+ r = requests.post(url, data)
+
+ # print json.dumps(r.json(), indent=2)
+
+ if r.status_code == 200:
+ print 'PUT (%s) in %sms' % (r.status_code, total_milliseconds(r.elapsed))
+ return r.json()
+
+ raise Exception('HTTP %s calling POST URL=[%s]: %s' % (r.status_code, url, r.text))
+
+ def delete(self, index):
+ url = '%s%s' % (self.base_url, index)
+ r = requests.delete(url)
+
+ if r.status_code == 200:
+ print 'DELETE (%s) in %sms' % (r.status_code, total_milliseconds(r.elapsed))
+ return r.json()
+
+ raise Exception('HTTP %s calling DELETE URL=[%s]: %s' % (r.status_code, url, r.text))
+
+ def create_index(self, name=None, shards=18 * 3, replicas=1):
+ data = {
+ "settings": {
+ "index": {
+ "action": {
+ "write_consistency": "one"
+ },
+ "number_of_shards": shards,
+ "number_of_replicas": replicas
+ }
+ }
+ }
+
+ try:
+ print 'Creating index %s' % name
+ response = self.put('/%s/' % name.lower(), data)
+
+ print response
+
+ except Exception, e:
+ print traceback.format_exc()
+
+ def delete_index(self, name):
+ try:
+ response = self.delete('/%s/' % name.lower())
+
+ print response
+
+ except Exception, e:
+ print traceback.format_exc()
+
+ def define_type_mapping(self, index_name, type_name):
+ try:
+ url = '/%s/_mapping/%s' % (index_name, type_name)
+ print url
+
+ response = self.put(url, get_type_mapping(type_name))
+
+ print response
+
+ except Exception, e:
+ print traceback.format_exc()
+
+
+class Worker(Process):
+ def __init__(self, work_queue):
+ super(Worker, self).__init__()
+ self.api_client = APIClient('http://%s:9200' % es_hosts[random.randint(0, len(es_hosts) - 1)].get('host'))
+ self.work_queue = work_queue
+ self.es = Elasticsearch(es_hosts)
+ self.sentence_list = loremipsum.get_sentences(1000)
+ self.re_first_word = re.compile('([A-z]+)')
+
+ def run(self):
+ print 'Starting %s ' % self.name
+ counter = 0
+
+ batch = []
+
+ while True:
+ index_batch_size = args.get('batch_size')
+ task = self.work_queue.get(timeout=600)
+ counter += 1
+
+ document = self.generate_document(task['field_count'])
+ flattened_doc = self.process_document(document,
+ task['type'],
+ task['uuid'],
+ task['uuid'])
+
+ index_type_tuple = (task['index'], task['type'])
+
+ # self.handle_document(task['index'], task['type'], task['uuid'], flattened_doc)
+
+ batch.append((index_type_tuple, flattened_doc))
+
+ if len(batch) >= index_batch_size:
+ self.handle_batch(batch)
+ batch = []
+
+ self.work_queue.task_done()
+
+ def generate_document(self, fields):
+
+ doc = {}
+
+ my_bool = True
+
+ for i in xrange(fields):
+ sentence_index = random.randint(0, max((fields / 2) - 1, 1))
+ sentence = self.sentence_list[sentence_index]
+
+ if random.random() >= .5:
+ key = self.re_first_word.findall(sentence)[1]
+ else:
+ key = self.re_first_word.findall(sentence)[1] + str(i)
+
+ field_type = random.random()
+
+ if field_type <= 0.3:
+ doc[key] = sentence
+
+ elif field_type <= 0.5:
+ doc[key] = random.randint(1, 1000000)
+
+ elif field_type <= 0.6:
+ doc[key] = random.random() * 1000000000
+
+ elif field_type == 0.7:
+ doc[key] = my_bool
+ my_bool = not my_bool
+
+ elif field_type == 0.8:
+ doc[key] = self.generate_document(max(fields / 5, 1))
+
+ elif field_type <= 1.0:
+ doc['mylocation'] = self.generate_location()
+
+ return doc
+
+ @staticmethod
+ def get_fields(document, base_name=None):
+ fields = []
+
+ for name, value in document.iteritems():
+ if base_name:
+ field_name = '%s.%s' % (base_name, name)
+ else:
+ field_name = name
+
+ if isinstance(value, dict):
+ fields += Worker.get_fields(value, field_name)
+ else:
+ value_name = None
+ if isinstance(value, basestring):
+ value_name = 'string'
+
+ elif isinstance(value, bool):
+ value_name = 'boolean'
+
+ elif isinstance(value, (int, long)):
+ value_name = 'long'
+
+ elif isinstance(value, float):
+ value_name = 'double'
+
+ if value_name:
+ field = {
+ 'name': field_name,
+ value_name: value
+ }
+ else:
+ field = {
+ 'name': field_name,
+ 'string': str(value)
+ }
+
+ fields.append(field)
+
+ return fields
+
+
+ @staticmethod
+ def process_document(document, doc_type, application_id, uuid):
+ response = {
+ 'entityId': uuid,
+ 'entityVersion': '1',
+ 'entityType': doc_type,
+ 'applicationId': application_id,
+ 'fields': Worker.get_fields(document)
+ }
+
+ return response
+
+ def handle_document(self, index, doc_type, uuid, document):
+
+ res = self.es.create(index=index,
+ doc_type=doc_type,
+ id=uuid,
+ body=document)
+
+ print res
+
+ def generate_location(self):
+ response = {}
+
+ lat = random.random() * 90.0
+ lon = random.random() * 180.0
+
+ lat_neg_true = True if lon > .5 else False
+ lon_neg_true = True if lat > .5 else False
+
+ lat = lat * -1.0 if lat_neg_true else lat
+ lon = lon * -1.0 if lon_neg_true else lon
+
+ response['location'] = {
+ 'lat': lat,
+ 'lon': lon
+ }
+
+ return response
+
+ def handle_batch(self, batch):
+ print 'HANDLE BATCH size=%s' % len(batch)
+ # self.api_client.define_type_mapping(index, doc_type)
+ self.api_client.index_batch(batch)
+
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+def get_type_mapping(type_name):
+ return {
+ type_name: {
+ "_routing": {
+ "path": "entityId",
+ "required": True
+ },
+ "properties": {
+ "entityId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "entityVersion": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "entityType": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "applicationId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "nodeId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "edgeName": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "entityNodeType": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "edgeTimestamp": {
+ "type": "long",
+ "doc_values": True
+ },
+ "edgeSearch": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "fields": {
+ "type": "nested",
+ "properties": {
+ "name": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "boolean": {
+ "type": "boolean",
+ "doc_values": True
+ },
+ "long": {
+ "type": "long",
+ "doc_values": True
+ },
+ "double": {
+ "type": "double",
+ "doc_values": True
+ },
+ "location": {
+ "type": "geo_point",
+ "lat_lon": True,
+ "geohash": True,
+ "doc_values": True
+ },
+ "string": {
+ "type": "string",
+ "norms": {
+ "enabled": False
+ },
+ "fields": {
+ "exact": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ }
+ }
+ },
+ "uuid": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ }
+ }
+ }
+ },
+ "_all": {
+ "enabled": False
+ }
+
+ }
+ }
+
+
+def main():
+ INDEX_COUNT = args.get('index_count')
+ TYPE_COUNT = args.get('type_count')
+ SETUP = args.get('setup')
+
+ indices = []
+ types = []
+ work_queue = JoinableQueue()
+
+ apiclient = APIClient('http://%s:9200' % es_hosts[random.randint(0, len(es_hosts) - 1)].get('host'))
+
+ workers = [Worker(work_queue) for x in xrange(args.get('workers'))]
+ [worker.start() for worker in workers]
+
+ try:
+ #
+ for x in xrange(TYPE_COUNT):
+ type_name = '%s_%s' % (args.get('type_prefix'), x)
+ types.append(type_name)
+
+ for x in xrange(INDEX_COUNT):
+ index_name = '%s_%s' % (args.get('index_prefix'), x)
+ indices.append(index_name)
+
+ if SETUP:
+ print 'Running setup...'
+
+ for index_name in indices:
+ apiclient.delete_index(index_name)
+
+ time.sleep(1)
+
+ for index_name in indices:
+ apiclient.create_index(
+ index_name,
+ shards=args['shard_count'],
+ replicas=args['replica_count'])
+
+ # time.sleep(5)
+
+ # for index_name in indices:
+ # for type_name in types:
+ # apiclient.define_type_mapping(index_name, type_name)
+
+ # time.sleep(5)
+
+ total_messages = args.get('document_count')
+ batch_size = 100000
+ message_counter = 0
+ fields = random.randint(50, 100)
+
+ while message_counter < total_messages:
+
+ for count in xrange(batch_size):
+
+ for index_name in indices:
+ doc_id = str(uuid.uuid1())
+
+ task = {
+ 'field_count': fields,
+ 'uuid': doc_id,
+ 'index': index_name,
+ 'type': types[random.randint(0, len(types) - 1)]
+ }
+
+ work_queue.put(task)
+
+ print 'Joining queue counter=[%s]...' % message_counter
+ work_queue.join()
+ print 'Done queue counter=[%s]...' % message_counter
+ message_counter += batch_size
+
+ except KeyboardInterrupt:
+ [worker.terminate() for worker in workers]
+
+
+main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
new file mode 100644
index 0000000..e3afdc3
--- /dev/null
+++ b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
@@ -0,0 +1,547 @@
+import json
+from multiprocessing import JoinableQueue, Process
+import random
+import re
+import traceback
+import uuid
+import time
+import sys
+
+import argparse
+import loremipsum
+import requests
+from elasticsearch import Elasticsearch
+
+es_hosts = [
+ {'host': 'ees000wo', 'port': 9200},
+ {'host': 'ees001wo', 'port': 9200},
+ {'host': 'ees002wo', 'port': 9200},
+ {'host': 'ees003wo', 'port': 9200},
+ {'host': 'ees004wo', 'port': 9200},
+ {'host': 'ees005wo', 'port': 9200},
+ {'host': 'ees006wo', 'port': 9200},
+ {'host': 'ees007wo', 'port': 9200},
+ {'host': 'ees008wo', 'port': 9200},
+ {'host': 'ees009wo', 'port': 9200},
+ {'host': 'ees010wo', 'port': 9200},
+ {'host': 'ees011wo', 'port': 9200},
+ {'host': 'ees012wo', 'port': 9200},
+ {'host': 'ees013wo', 'port': 9200},
+ {'host': 'ees014wo', 'port': 9200},
+ {'host': 'ees015wo', 'port': 9200},
+ {'host': 'ees016wo', 'port': 9200},
+ {'host': 'ees017wo', 'port': 9200}
+]
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='ElasticSearch Index Test 1')
+
+ parser.add_argument('-t', '--type_count',
+ help='The number of types to produce',
+ type=int,
+ default=50)
+
+ parser.add_argument('-ic', '--index_count',
+ help='The number of indices to create',
+ type=int,
+ default=50)
+
+ parser.add_argument('-sc', '--shard_count',
+ help='The number of indices to create',
+ type=int,
+ default=50)
+
+ parser.add_argument('-rc', '--replica_count',
+ help='The number of indices to create',
+ type=int,
+ default=1)
+
+ parser.add_argument('-w', '--workers',
+ help='The number of worker threads',
+ type=int,
+ default=8)
+
+ parser.add_argument('-dc', '--document_count',
+ help='The number of documents per index',
+ type=long,
+ default=100000000)
+
+ parser.add_argument('-bs', '--batch_size',
+ help='The size of batches to send to ES',
+ type=long,
+ default=25)
+
+ parser.add_argument('-ip', '--index_prefix',
+ help='The Prefix to use for index names',
+ type=str,
+ default='apigee_ftw')
+
+ parser.add_argument('-tp', '--type_prefix',
+ help='The Prefix to use for type names',
+ type=str,
+ default='type_this')
+
+ parser.add_argument('-s', '--setup',
+ help='The Prefix to use for type names',
+ action='store_true')
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+args = parse_args()
+
+
+class APIClient():
+ def __init__(self, base_url):
+ self.base_url = base_url
+
+ def put(self, path='/', data=None):
+ if not data:
+ data = {}
+
+ url = '%s%s' % (self.base_url, path)
+ r = requests.put(url, json.dumps(data))
+
+ if r.status_code == 200:
+ print 'PUT (%s) in %sms' % (r.status_code, total_milliseconds(r.elapsed))
+ return r.json()
+
+ raise Exception('HTTP %s calling PUT on URL=[%s]: %s' % (r.status_code, url, r.text))
+
+ def index_docs(self, index, documents, type):
+
+ data = ''
+
+ for doc in documents:
+ data += '{ "index" : { "_index" : "%s", "_type" : "%s", "_id" : "%s" } }\n' % (index, type, doc['entityId'])
+ data += json.dumps(doc)
+ data += '\n'
+
+ url = '%s/_bulk' % self.base_url
+
+ # print data
+
+ r = requests.post(url, data)
+
+ # print json.dumps(r.json(), indent=2)
+
+ if r.status_code == 200:
+ print 'PUT (%s) in %sms' % (r.status_code, total_milliseconds(r.elapsed))
+ return r.json()
+
+ raise Exception('HTTP %s calling POST URL=[%s]: %s' % (r.status_code, url, r.text))
+
+ def delete(self, index):
+ url = '%s%s' % (self.base_url, index)
+ r = requests.delete(url)
+
+ if r.status_code == 200:
+ print 'DELETE (%s) in %sms' % (r.status_code, total_milliseconds(r.elapsed))
+ return r.json()
+
+ raise Exception('HTTP %s calling DELETE URL=[%s]: %s' % (r.status_code, url, r.text))
+
+ def create_index(self, name=None, shards=18 * 3, replicas=1):
+ data = {
+ "settings": {
+ "index": {
+ "action": {
+ "write_consistency": "one"
+ },
+ "number_of_shards": shards,
+ "number_of_replicas": replicas
+ }
+ }
+ }
+
+ try:
+ print 'Creating index %s' % name
+ response = self.put('/%s/' % name.lower(), data)
+
+ print response
+
+ except Exception, e:
+ print traceback.format_exc()
+
+ def delete_index(self, name):
+ try:
+ response = self.delete('/%s/' % name.lower())
+
+ print response
+
+ except Exception, e:
+ print traceback.format_exc()
+
+ def define_type_mapping(self, index_name, type_name):
+ try:
+ url = '/%s/_mapping/%s' % (index_name, type_name)
+ print url
+
+ response = self.put(url, get_type_mapping(type_name))
+
+ print response
+
+ except Exception, e:
+ print traceback.format_exc()
+
+
+class Worker(Process):
+ def __init__(self, work_queue):
+ super(Worker, self).__init__()
+ self.api_client = APIClient('http://%s:9200' % es_hosts[random.randint(0, len(es_hosts) - 1)].get('host'))
+ self.work_queue = work_queue
+ self.es = Elasticsearch(es_hosts)
+ self.sentence_list = loremipsum.get_sentences(1000)
+ self.re_first_word = re.compile('([A-z]+)')
+
+ def run(self):
+ print 'Starting %s ' % self.name
+ counter = 0
+
+ docs = {}
+
+ while True:
+ index_batch_size = args.get('batch_size')
+ task = self.work_queue.get(timeout=600)
+ counter += 1
+
+ document = self.generate_document(task['field_count'])
+ flattened_doc = self.process_document(document,
+ task['type'],
+ task['uuid'],
+ task['uuid'])
+
+ index_type_tuple = (task['index'], task['type'])
+
+ # self.handle_document(task['index'], task['type'], task['uuid'], flattened_doc)
+
+ doc_array = docs.get(index_type_tuple)
+
+ if doc_array is None:
+ doc_array = []
+ docs[index_type_tuple] = doc_array
+
+ doc_array.append(flattened_doc)
+
+ if len(doc_array) >= index_batch_size:
+ self.handle_batch(task['index'], task['type'], doc_array)
+ doc_array = []
+
+ self.work_queue.task_done()
+
+ def generate_document(self, fields):
+
+ doc = {}
+
+ my_bool = True
+
+ for i in xrange(fields):
+ sentence_index = random.randint(0, max((fields / 2) - 1, 1))
+ sentence = self.sentence_list[sentence_index]
+
+ if random.random() >= .5:
+ key = self.re_first_word.findall(sentence)[1]
+ else:
+ key = self.re_first_word.findall(sentence)[1] + str(i)
+
+ field_type = random.random()
+
+ if field_type <= 0.3:
+ doc[key] = sentence
+
+ elif field_type <= 0.5:
+ doc[key] = random.randint(1, 1000000)
+
+ elif field_type <= 0.6:
+ doc[key] = random.random() * 1000000000
+
+ elif field_type == 0.7:
+ doc[key] = my_bool
+ my_bool = not my_bool
+
+ elif field_type == 0.8:
+ doc[key] = self.generate_document(max(fields / 5, 1))
+
+ elif field_type <= 1.0:
+ doc['mylocation'] = self.generate_location()
+
+ return doc
+
+ @staticmethod
+ def get_fields(document, base_name=None):
+ fields = []
+
+ for name, value in document.iteritems():
+ if base_name:
+ field_name = '%s.%s' % (base_name, name)
+ else:
+ field_name = name
+
+ if isinstance(value, dict):
+ fields += Worker.get_fields(value, field_name)
+ else:
+ value_name = None
+ if isinstance(value, basestring):
+ value_name = 'string'
+
+ elif isinstance(value, bool):
+ value_name = 'boolean'
+
+ elif isinstance(value, (int, long)):
+ value_name = 'long'
+
+ elif isinstance(value, float):
+ value_name = 'double'
+
+ if value_name:
+ field = {
+ 'name': field_name,
+ value_name: value
+ }
+ else:
+ field = {
+ 'name': field_name,
+ 'string': str(value)
+ }
+
+ fields.append(field)
+
+ return fields
+
+ @staticmethod
+ def process_document(document, doc_type, application_id, uuid):
+ response = {
+ 'entityId': uuid,
+ 'entityVersion': '1',
+ 'entityType': doc_type,
+ 'applicationId': application_id,
+ 'fields': Worker.get_fields(document)
+ }
+
+ return response
+
+ def handle_document(self, index, doc_type, uuid, document):
+
+ res = self.es.create(index=index,
+ doc_type=doc_type,
+ id=uuid,
+ body=document)
+
+ print res
+
+ def generate_location(self):
+ response = {}
+
+ lat = random.random() * 90.0
+ lon = random.random() * 180.0
+
+ lat_neg_true = True if lon > .5 else False
+ lon_neg_true = True if lat > .5 else False
+
+ lat = lat * -1.0 if lat_neg_true else lat
+ lon = lon * -1.0 if lon_neg_true else lon
+
+ response['location'] = {
+ 'lat': lat,
+ 'lon': lon
+ }
+
+ return response
+
+ def handle_batch(self, index, doc_type, docs):
+ print 'HANDLE BATCH'
+ self.api_client.define_type_mapping(index, doc_type)
+ self.api_client.index_docs(index, docs, doc_type)
+
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+def get_type_mapping(type_name):
+ return {
+ type_name: {
+ "_routing": {
+ "path": "entityId",
+ "required": True
+ },
+ "properties": {
+ "entityId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "entityVersion": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "entityType": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "applicationId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "nodeId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "edgeName": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "entityNodeType": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "edgeTimestamp": {
+ "type": "long",
+ "doc_values": True
+ },
+ "edgeSearch": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "fields": {
+ "type": "nested",
+ "properties": {
+ "name": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ },
+ "boolean": {
+ "type": "boolean",
+ "doc_values": True
+ },
+ "long": {
+ "type": "long",
+ "doc_values": True
+ },
+ "double": {
+ "type": "double",
+ "doc_values": True
+ },
+ "location": {
+ "type": "geo_point",
+ "lat_lon": True,
+ "geohash": True,
+ "doc_values": True
+ },
+ "string": {
+ "type": "string",
+ "norms": {
+ "enabled": False
+ },
+ "fields": {
+ "exact": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ }
+ }
+ },
+ "uuid": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": True
+ }
+ }
+ }
+ },
+ "_all": {
+ "enabled": False
+ }
+
+ }
+ }
+
+
+def main():
+ INDEX_COUNT = args.get('index_count')
+ TYPE_COUNT = args.get('type_count')
+ SETUP = args.get('setup')
+
+ indices = []
+ types = []
+ work_queue = JoinableQueue()
+
+ apiclient = APIClient('http://%s:9200' % es_hosts[random.randint(1, len(es_hosts) - 1)].get('host'))
+
+ workers = [Worker(work_queue) for x in xrange(args.get('workers'))]
+ [worker.start() for worker in workers]
+
+ try:
+ #
+ for x in xrange(TYPE_COUNT):
+ type_name = '%s_%s' % (args.get('type_prefix'), x)
+ types.append(type_name)
+
+ for x in xrange(INDEX_COUNT):
+ index_name = '%s_%s' % (args.get('index_prefix'), x)
+ indices.append(index_name)
+
+ if SETUP:
+ print 'Running setup...'
+
+ for index_name in indices:
+ apiclient.delete_index(index_name)
+
+ time.sleep(5)
+
+ for index_name in indices:
+ apiclient.create_index(
+ index_name,
+ shards=args['shard_count'],
+ replicas=args['replica_count'])
+
+ # time.sleep(5)
+
+ # for index_name in indices:
+ # for type_name in types:
+ # apiclient.define_type_mapping(index_name, type_name)
+
+ # time.sleep(5)
+
+ total_messages = args.get('document_count')
+ batch_size = 100000
+ message_counter = 0
+ fields = random.randint(50, 100)
+
+ while message_counter < total_messages:
+
+ for count in xrange(batch_size):
+
+ for index_name in indices:
+ doc_id = str(uuid.uuid1())
+
+ task = {
+ 'field_count': fields,
+ 'uuid': doc_id,
+ 'index': index_name,
+ 'type': types[random.randint(0, len(types) - 1)]
+ }
+
+ work_queue.put(task)
+
+ print 'Joining queue counter=[%s]...' % message_counter
+ work_queue.join()
+ print 'Done queue counter=[%s]...' % message_counter
+ message_counter += batch_size
+
+ except KeyboardInterrupt:
+ [worker.terminate() for worker in workers]
+
+
+main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/requirements.txt
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/requirements.txt b/utils/usergrid-util-python/requirements.txt
new file mode 100644
index 0000000..d15d7be
--- /dev/null
+++ b/utils/usergrid-util-python/requirements.txt
@@ -0,0 +1,4 @@
+urllib3
+usergrid
+requests
+redis
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
new file mode 100644
index 0000000..ce38544
--- /dev/null
+++ b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
@@ -0,0 +1,132 @@
+# docs page: http://docs.apigee.com/api-baas/content/creating-activity
+
+# create user 1
+# post event for user 1
+# check feed for user 1
+
+# create user 2
+# user 2 follows user 1
+# post event for user 1
+
+# check feed for user 1
+# check feed for user 2
+import json
+
+import requests
+
+collection_url_template = "{api_url}/{org}/{app}/{collection}"
+entity_url_template = "{api_url}/{org}/{app}/{collection}/{entity_id}"
+connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}"
+connection_create_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}"
+
+user_url_template = "{api_url}/{org}/{app}/users/{username}"
+user_feed_url_template = "{api_url}/{org}/{app}/users/{username}/feed"
+user_activity_url_template = "{api_url}/{org}/{app}/users/{username}/activities"
+user_follows_url_template = "{api_url}/{org}/{app}/users/{user2}/following/users/{user1}"
+
+url_data = {
+ 'api_url': 'https://amer-apibaas-prod.apigee.net/appservices',
+ 'org': 'jwest-samples',
+ 'app': 'feed-example'
+}
+
+session = requests.Session()
+
+
+def create_user(user):
+ data = {
+ 'username': user,
+ 'email': '%s@example.com' % user
+ }
+
+ url = collection_url_template.format(collection='users', **url_data)
+
+ r = session.post(url, json.dumps(data))
+
+ if r.status_code != 200:
+ print 'Error creating user [%s] at URL=[%s]: %s' % (user, url, r.text)
+
+
+def post_activity(user, text):
+ activity = {
+ "actor": {
+ "displayName": user,
+ "username": user,
+ "image": {
+ "duration": 0,
+ "height": 80,
+ "url": "http://www.gravatar.com/avatar/", "width": 80},
+ "email": "%s@example.com" % user
+ },
+ "verb": "post",
+ "content": text
+ }
+
+ url = user_activity_url_template.format(username=user, **url_data)
+
+ r = session.post(url, json.dumps(activity))
+
+ if r.status_code != 200:
+ print 'Error creating activity for user [%s] at URL=[%s]: %s' % (user, url, r.text)
+
+
+def get_feed(user):
+ url = user_feed_url_template.format(username=user, **url_data)
+
+ r = session.get(url)
+
+ if r.status_code != 200:
+ print 'Error getting feed for user [%s] at URL=[%s]: %s' % (user, url, r.text)
+
+ else:
+ print '----- START'
+ print json.dumps(r.json(), indent=2)
+ print '----- END'
+
+
+def create_follows(user, user_to_follow):
+ url = user_follows_url_template.format(user1=user, user2=user_to_follow, **url_data)
+
+ r = session.post(url)
+
+ print r.text
+
+ if r.status_code != 200:
+ print 'Error getting creating follows from user [%s] to user [%s] at URL=[%s]: %s' % (
+ user, user_to_follow, url, r.text)
+
+
+def delete_user(username):
+ url = user_url_template.format(username=username, **url_data)
+
+ r = session.post(url)
+
+ # print r.text
+
+ if r.status_code != 200:
+ print 'Error deleting user [%s] at URL=[%s]: %s' % (username, url, r.text)
+
+
+user_base = 'natgeo'
+
+user1 = '%s_%s' % (user_base, 1)
+user2 = '%s_%s' % (user_base, 2)
+
+create_user(user1)
+post_activity(user1, 'Hello World!')
+
+get_feed(user1)
+
+create_user(user2)
+create_follows(user2, user1)
+post_activity(user2, "I'm here!")
+get_feed(user2)
+
+post_activity(user1, 'SEE YA!!')
+
+get_feed(user2)
+
+get_feed(user1)
+
+delete_user(user1)
+delete_user(user2)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/samples/beacon-event-example.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/beacon-event-example.py b/utils/usergrid-util-python/samples/beacon-event-example.py
new file mode 100644
index 0000000..fc05cdc
--- /dev/null
+++ b/utils/usergrid-util-python/samples/beacon-event-example.py
@@ -0,0 +1,196 @@
+# URL Templates for Usergrid
+import json
+import random
+
+import requests
+from multiprocessing import Process, Pool
+
+import time
+
+collection_url_template = "{api_url}/{org}/{app}/{collection}"
+entity_url_template = "{api_url}/{org}/{app}/{collection}/{entity_id}"
+connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}"
+connection_create_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}"
+
+url_data = {
+ 'api_url': 'https://amer-apibaas-prod.apigee.net/appservices',
+ 'org': 'jwest-samples',
+ 'app': 'event-example'
+}
+
+url_data = {
+ 'api_url': 'http://usergrid_app.cfapps-01.haas-26.pez.pivotal.io',
+ 'org': 'jwest',
+ 'app': 'sandbox'
+}
+
+session = requests.Session()
+
+
+class EventGenerator(Process):
+ def __init__(self, store_id, event_count, user_array, beacons):
+ super(EventGenerator, self).__init__()
+
+ self.store_id = store_id
+ self.user_array = user_array
+ self.event_count = event_count
+ self.beacons = beacons
+ self.session = requests.Session()
+ self.create_store(self.store_id)
+ self.create_users(self.user_array)
+
+ def create_store(self, store_id):
+ url = entity_url_template.format(collection='stores', entity_id=store_id, **url_data)
+
+ r = self.session.put(url, data=json.dumps({"name": store_id}))
+
+ if r.status_code != 200:
+ print 'Error creating store [%s] at URL=[%s]: %s' % (store_id, url, r.text)
+
+ def create_event(self, user, event):
+ print 'creating event: %s' % json.dumps(event)
+
+ url = collection_url_template.format(collection='general-events', **url_data)
+
+ r = self.session.post(url, data=json.dumps(event))
+
+ if r.status_code == 200:
+ res = r.json()
+ entity = res.get('entities')[0]
+ event_uuid = entity.get('uuid')
+
+ # link to user
+ create_connection_url = connection_create_url_template.format(collection='users',
+ uuid=user,
+ verb='events',
+ target_uuid=event_uuid,
+ **url_data)
+
+ r_connect = self.session.post(create_connection_url)
+
+ if r_connect.status_code == 200:
+ print 'created connection: %s' % create_connection_url
+
+ # link to store
+ create_connection_url = connection_create_url_template.format(collection='stores',
+ uuid=event.get('storeId'),
+ verb='events',
+ target_uuid=event_uuid,
+ **url_data)
+
+ r_connect = self.session.post(create_connection_url)
+
+ if r_connect.status_code == 200:
+ print 'created connection: %s' % create_connection_url
+
+ if event.get('eventType') == 'beacon':
+ # link to beacon
+ create_connection_url = connection_create_url_template.format(collection='beacons',
+ uuid=event.get('beaconId'),
+ verb='events',
+ target_uuid=event_uuid,
+ **url_data)
+
+ r_connect = self.session.post(create_connection_url)
+
+ if r_connect.status_code == 200:
+ print 'created connection: %s' % create_connection_url
+ else:
+ print 'Error creating connection at URL=[%s]: %s' % (create_connection_url, r.text)
+
+ def run(self):
+
+ for user in self.user_array:
+
+ # store 123
+ self.create_event(user, {
+ 'storeId': self.store_id,
+ 'eventType': 'enterStore'
+ })
+
+ for x in xrange(0, self.event_count):
+ beacon_number = random.randint(0, len(self.beacons) - 1)
+ beacon_name = self.beacons[beacon_number]
+
+ event = {
+ 'beaconId': '%s-%s' % (self.store_id, beacon_name),
+ 'storeId': self.store_id,
+ 'eventType': 'beacon'
+ }
+
+ self.create_event(user, event)
+
+ self.create_event(user, {
+ 'storeId': self.store_id,
+ 'eventType': 'exitStore'
+ })
+
+ def create_users(self, user_array):
+ for user in user_array:
+ self.create_user(user)
+
+ def create_user(self, user):
+ data = {
+ 'username': user,
+ 'email': '%s@example.com' % user
+ }
+
+ url = collection_url_template.format(collection='users', **url_data)
+
+ r = self.session.post(url, json.dumps(data))
+
+ if r.status_code != 200:
+ print 'Error creating user [%s] at URL=[%s]: %s' % (user, url, r.text)
+
+
+def create_entity(entity_type, entity_name):
+ url = entity_url_template.format(collection=entity_type, entity_id=entity_name, **url_data)
+ r = session.put(url, data=json.dumps({'name': entity_name}))
+
+ if r.status_code != 200:
+ print 'Error creating %s [%s] at URL=[%s]: %s' % (entity_type, entity_name, url, r.text)
+
+
+def create_beacon(beacon_name):
+ create_entity('beacons', beacon_name)
+
+
+def create_store(store_name):
+ create_entity('stores', store_name)
+
+
+def main():
+ beacons = ["b1", "b2", "b3", "b4", "b5", "b6"]
+
+ stores = ['store_123', 'store_456', 'store_789', 'store_901']
+
+ beacon_names = []
+
+ for store in stores:
+ for beacon in beacons:
+ beacon_names.append('%s-%s' % (store, beacon))
+
+ pool = Pool(16)
+
+ pool.map(create_beacon, beacon_names)
+ pool.map(create_store, stores)
+
+ processes = [
+ EventGenerator(stores[0], 100, ['jeff', 'julie'], beacons=beacons),
+ EventGenerator(stores[0], 100, ['russo', 'dunker'], beacons=beacons),
+ EventGenerator(stores[2], 100, ['jeff', 'julie'], beacons=beacons),
+ EventGenerator(stores[2], 100, ['russo', 'dunker'], beacons=beacons),
+ EventGenerator(stores[3], 100, ['jeff', 'julie'], beacons=beacons),
+ EventGenerator(stores[3], 100, ['russo', 'dunker'], beacons=beacons),
+ EventGenerator(stores[1], 100, ['bala', 'shankar'], beacons=beacons),
+ EventGenerator(stores[1], 100, ['chet', 'anant'], beacons=beacons)
+ ]
+
+ [p.start() for p in processes]
+
+ while len([p for p in processes if p.is_alive()]) > 0:
+ print 'Processors active, waiting'
+ time.sleep(1)
+
+
+main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/samples/counter_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/counter_test.py b/utils/usergrid-util-python/samples/counter_test.py
new file mode 100644
index 0000000..7852b26
--- /dev/null
+++ b/utils/usergrid-util-python/samples/counter_test.py
@@ -0,0 +1,31 @@
+import datetime
+import time
+import json
+
+import requests
+
+tstamp = time.gmtime() * 1000
+
+s = requests.Session()
+
+s.headers.update({'authorization': 'Bearer YWMt7AHANAKcEeaVR-EahuX8EgAAAVQ7Q56jxQjUsmhJn8rGLTth0XtRrBSIzDA'})
+s.headers.update({'content-type': 'application/json'})
+
+url = 'https://host/appservices-new/usergrid/pushtest/events'
+
+body = {
+ "timestamp": tstamp,
+ "counters": {
+ "counters.jeff.west": 1
+ }
+}
+
+r = s.post(url, data=json.dumps(body))
+
+print r.status_code
+
+time.sleep(30)
+
+r = s.get('https://host/appservices-new/usergrid/pushtest//counters?counter=counters.jeff.west')
+
+print r.text
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/setup.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/setup.py b/utils/usergrid-util-python/setup.py
new file mode 100755
index 0000000..1f19cb2
--- /dev/null
+++ b/utils/usergrid-util-python/setup.py
@@ -0,0 +1,40 @@
+from setuptools import setup, find_packages
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+VERSION = '0.5.13'
+
+setup(
+ name='usergrid-tools',
+ version=VERSION,
+ description='Tools for working with Apache Usergrid',
+ url='http://usergrid.apache.org',
+ download_url="https://codeload.github.com/jwest-apigee/usergrid-util-python/zip/%s" % VERSION,
+ author='Jeff West',
+ author_email='jwest@apigee.com',
+
+ # packages=['usergrid_tools', 'es_tools'],
+ packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests", "sandbox"]),
+
+ install_requires=[
+ 'requests',
+ 'usergrid>=0.1.3',
+ 'time_uuid',
+ 'argparse',
+ 'redis',
+ 'ConcurrentLogHandler',
+ ],
+
+ entry_points={
+ 'console_scripts': [
+ 'usergrid_iterator = usergrid_tools.iterators.simple_iterator:main',
+ 'usergrid_data_migrator = usergrid_tools.migration.usergrid_data_migrator:main',
+ 'usergrid_data_exporter = usergrid_tools.migration.usergrid_data_exporter:main',
+ 'usergrid_entity_index_test = usergrid_tools.indexing.entity_index_test:main',
+ 'usergrid_batch_index_test = usergrid_tools.indexing.batch_index_test:main',
+ 'usergrid_parse_importer = usergrid_tools.parse_importer.parse_importer:main',
+ 'usergrid_deleter = usergrid_tools.parse_importer.parse_importer:main',
+ 'usergrid_library_check = usergrid_tools.library_check:main',
+ ]
+ }
+)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/__init__.py b/utils/usergrid-util-python/usergrid_tools/__init__.py
new file mode 100644
index 0000000..beed654
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/__init__.py
@@ -0,0 +1,4 @@
+import migration
+import iterators
+import indexing
+import general
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/general/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/__init__.py b/utils/usergrid-util-python/usergrid_tools/general/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/general/deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/deleter.py b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
new file mode 100644
index 0000000..3c53cae
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
@@ -0,0 +1,151 @@
+import json
+import traceback
+import requests
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+# for Apigee Developer, leave this as is. For paid BaaS instances change this to https://{your_api_url}/[appservices]
+api_url = 'https://api.usergrid.com'
+
+# specify the org[] / app[] / collection[] to delete
+# Org and App level are required. If no collections are specified, all collections will be deleted
+# you also need to specify the client_id and secret of each org
+
+data_map = {
+ "orgs":
+ {
+ "myOrg": {
+ "apps": {
+ "myApp": {
+ "collections": [
+ 'examples'
+ ]
+ }
+ },
+ "credentials": {
+ "client_id": "foo",
+ "client_secret": "bar"
+ }
+ }
+ }
+}
+# it is generally not a good idea to delete more than 100 at a time due to latency and resource utilization
+url_template = '{api_url}/{org}/{app}/{collection}?limit=250'
+
+session = requests.Session()
+
+
+def check_response_status(response, message='', exit_on_error=True):
+ if response.status_code != 200:
+ print 'ERROR: ' + message
+ print response.text
+
+ if exit_on_error:
+ exit()
+
+
+def delete_all_collections(org, app, token):
+ url = '{api_url}/{org}/{app}'.format(api_url=api_url, org=org, app=app)
+
+ print 'Listing collections at URL: %s' % url
+
+ r = session.get(url)
+
+ if r.status_code != 200:
+ print r.text
+
+ collections = []
+
+ delete_collections(org, app, collections, token)
+
+
+def delete_collections(org, app, collections, token):
+ print 'Deleting [%s] collections: %s' % (len(collections), collections)
+
+ for collection in collections:
+ print 'Deleting collection [%s]...' % collection
+
+ keep_going = True
+
+ count_with_zero = 0
+
+ while keep_going:
+
+ url = url_template.format(api_url=api_url, org=org, app=app, collection=collection)
+
+ try:
+ response = session.get(url)
+ check_response_status(response, message='Unable to GET URL: %s' % url)
+
+ count = len(response.json().get('entities'))
+ total_ms = total_milliseconds(response.elapsed)
+
+ print 'GET %s from collection %s in %s' % (count, collection, total_ms)
+ print 'Deleting...'
+
+ response = session.delete(url)
+
+ check_response_status(response, message='UNABLE TO DELETE on URL: %s' % url)
+
+ try:
+ count = len(response.json().get('entities'))
+ total_ms = total_milliseconds(response.elapsed)
+
+ print 'Deleted %s from collection %s in %s' % (count, collection, total_ms)
+
+ if count == 0:
+ count_with_zero += 1
+ print 'Count with ZERO: %s' % count_with_zero
+
+ # if there are 10 in a row with zero entities returned, we're done
+ if count_with_zero >= 10:
+ keep_going = False
+ else:
+ count_with_zero = 0
+ except:
+ print 'Error! HTTP Status: %s response: %s' % (response.status_code, response.text)
+
+ except KeyboardInterrupt:
+ exit()
+
+ except:
+ print traceback.format_exc()
+
+
+# iterate the orgs specified in the configuration above
+for org, org_data in data_map.get('orgs', {}).iteritems():
+
+ credentials = org_data.get('credentials', {})
+
+ token_request = {
+ 'grant_type': 'client_credentials',
+ 'client_id': credentials.get('client_id'),
+ 'client_secret': credentials.get('client_secret'),
+ }
+
+ token_url = '{api_url}/management/token'.format(api_url=api_url)
+
+ r = session.post(token_url, data=json.dumps(token_request))
+
+ check_response_status(r, message='Unable to get Token at URL %s' % token_url)
+
+ token = r.json().get('access_token')
+ session.headers.update({'Authorization': 'Bearer ' + token})
+
+ # iterate the apps specified in the config above
+ for app, app_data in org_data.get('apps', {}).iteritems():
+
+ collections = app_data.get('collections', [])
+
+ # if the list of collections is empty, delete all collections
+ if len(collections) == 0:
+ delete_all_collections(org, app, token)
+
+ # Otherwise, delete the specified collections
+ else:
+ delete_collections(org, app, collections, token)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
new file mode 100644
index 0000000..3682d18
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
@@ -0,0 +1,25 @@
+from usergrid import UsergridQueryIterator
+
+### This iterates a collection using GRAPH and checks whether there are more than on entity with the same name
+
+url = 'https://host/org/app/collection?access_token=foo&limit=1000'
+
+q = UsergridQueryIterator(url)
+
+name_tracker = {}
+counter = 0
+for e in q:
+ counter += 1
+
+ if counter % 1000 == 1:
+ print 'Count: %s' % counter
+
+ name = e.get('name')
+
+ if name in name_tracker:
+ name_tracker[name].append(e.get('uuid'))
+
+ print 'duplicates for name=[%s]: %s' % (name, name_tracker[name])
+
+ else:
+ name_tracker[name] = [e.get('uuid')]
[12/50] [abbrv] usergrid git commit: made changes for PR comments..
Posted by mr...@apache.org.
made changes for PR comments..
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/930308d7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/930308d7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/930308d7
Branch: refs/heads/master
Commit: 930308d7e30f31f7d9f0f25f0bbb02982c30e86e
Parents: b583207
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Fri Jul 8 11:20:36 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Fri Jul 8 11:20:36 2016 -0700
----------------------------------------------------------------------
.../usergrid/rest/exceptions/AuthErrorInfo.java | 2 +-
.../rest/management/ManagementResource.java | 7 +++--
.../organizations/OrganizationsResource.java | 5 +---
.../rest/management/users/UserResource.java | 31 +++++++-------------
.../rest/management/users/UsersResource.java | 7 ++---
.../OAuth2AccessTokenSecurityFilter.java | 4 +--
...alSSOProviderAdminUserNotFoundException.java | 11 +++++++
.../tokens/cassandra/TokenServiceImpl.java | 13 +++++---
8 files changed, 40 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/930308d7/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java
index 8b7b969..c9149e5 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java
@@ -43,7 +43,7 @@ public enum AuthErrorInfo {
INVALID_USERNAME_OR_PASSWORD_ERROR( "auth_invalid_username_or_password",
"Unable to authenticate due to username or password being incorrect" ), //
UNVERIFIED_OAUTH_ERROR( "auth_unverified_oath", "Unable to authenticate OAuth credentials" ), //
- EXTERNALSSOPROVIDER_UNACTIVATED_ADMINUSER("externalssoprovider_unactivated_adminuser","Admin user needs to be activated via the external provider"),
+ EXTERNALSSOPROVIDER_UNACTIVATED_ADMINUSER("externalssoprovider_unactivated_adminuser","Admin user not found or does not have access to any organizations."),
NO_DOMAIN_ERROR( "auth_no_application", "Unable to authenticate due to application not found" ), //
NOT_DOMAIN_OWNER_ERROR( "auth_not_application_owner", "" ), //
EXPIRED_ACCESS_TOKEN_ERROR( "expired_token", "Unable to authenticate due to expired access token" ), //
http://git-wip-us.apache.org/repos/asf/usergrid/blob/930308d7/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 3d794d6..056303a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -531,10 +531,11 @@ public class ManagementResource extends AbstractContextResource {
return; // we only care about username/password auth
}
- final boolean externalTokensEnabled =
- !StringUtils.isEmpty( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
+ //why !isexternal_sso_enabled ?
+// final boolean externalTokensEnabled =
+// !StringUtils.isEmpty( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
- if ( externalTokensEnabled ) {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
// when external tokens enabled then only superuser can obtain an access token
http://git-wip-us.apache.org/repos/asf/usergrid/blob/930308d7/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
index 0e77d97..476e315 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
@@ -185,10 +185,7 @@ public class OrganizationsResource extends AbstractContextResource {
String email, String password, Map<String, Object> userProperties,
Map<String, Object> orgProperties, String callback ) throws Exception {
- final boolean externalTokensEnabled =
- !StringUtils.isEmpty( properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
-
- if ( externalTokensEnabled ) {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
throw new IllegalArgumentException( "Organization / Admin Users must be created via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/930308d7/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index b16e85c..739ef28 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -47,7 +47,6 @@ import java.util.UUID;
import static org.apache.usergrid.security.shiro.utils.SubjectUtils.isServiceAdmin;
import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER_URL;
-import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_SSO_ENABLED;
import static org.apache.usergrid.utils.ConversionUtils.string;
@@ -216,10 +215,7 @@ public class UserResource extends AbstractContextResource {
@Produces( MediaType.TEXT_HTML )
public Viewable showPasswordResetForm( @Context UriInfo ui, @QueryParam( "token" ) String token ) {
- final boolean externalTokensEnabled =
- Boolean.valueOf( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
-
- if ( externalTokensEnabled ) {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
throw new IllegalArgumentException( "Admin Users must reset passwords via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
}
@@ -263,10 +259,7 @@ public class UserResource extends AbstractContextResource {
logger.trace("handlePasswordResetForm");
}
- final boolean externalTokensEnabled =
- Boolean.valueOf( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
-
- if ( externalTokensEnabled ) {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
throw new IllegalArgumentException( "Admin Users must reset passwords via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
}
@@ -352,10 +345,7 @@ public class UserResource extends AbstractContextResource {
@Produces( MediaType.TEXT_HTML )
public Viewable activate( @Context UriInfo ui, @QueryParam( "token" ) String token ) {
- final boolean externalTokensEnabled =
- Boolean.valueOf( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
-
- if ( externalTokensEnabled ) {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
throw new IllegalArgumentException( "Admin Users must activate via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
}
@@ -385,10 +375,7 @@ public class UserResource extends AbstractContextResource {
@Produces( MediaType.TEXT_HTML )
public Viewable confirm( @Context UriInfo ui, @QueryParam( "token" ) String token ) {
- final boolean externalTokensEnabled =
- Boolean.valueOf( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
-
- if ( externalTokensEnabled ) {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
throw new IllegalArgumentException( "Admin Users must confirm via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
}
@@ -424,10 +411,7 @@ public class UserResource extends AbstractContextResource {
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
- final boolean externalTokensEnabled =
- Boolean.valueOf( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
-
- if ( externalTokensEnabled ) {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
throw new IllegalArgumentException( "Admin Users must reactivate via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
}
@@ -451,6 +435,11 @@ public class UserResource extends AbstractContextResource {
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
+ throw new IllegalArgumentException( "Admin Users must tokens must be revoked via " +
+ properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
+ }
+
UUID adminId = user.getUuid();
logger.info( "Revoking user tokens for {}", adminId );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/930308d7/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
index 9730e06..64281b6 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
@@ -44,7 +44,6 @@ import java.util.UUID;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.usergrid.rest.exceptions.SecurityException.mappableSecurityException;
-import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_SSO_ENABLED;
import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER_URL;
@@ -115,10 +114,7 @@ public class UsersResource extends AbstractContextResource {
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
- final boolean externalTokensEnabled =
- Boolean.valueOf( properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ) );
-
- if ( externalTokensEnabled ) {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
throw new IllegalArgumentException( "Admin Users must signup via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
}
@@ -141,6 +137,7 @@ public class UsersResource extends AbstractContextResource {
UserInfo user = null;
if ( tokens.isExternalSSOProviderEnabled() ){
+ //autoactivating user, since the activation
user = management.createAdminUser(null,username,name,email,password,true,false);
}
else {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/930308d7/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
index 4132dd3..7b35df6 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
@@ -27,7 +27,7 @@ import org.apache.shiro.subject.Subject;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.management.exceptions.ExternalSSOProviderAdminUserNotFoundExceptions;
+import org.apache.usergrid.management.exceptions.ExternalSSOProviderAdminUserNotFoundException;
import org.apache.usergrid.management.exceptions.ManagementException;
import org.apache.usergrid.security.AuthPrincipalInfo;
import org.apache.usergrid.security.AuthPrincipalType;
@@ -110,7 +110,7 @@ public class OAuth2AccessTokenSecurityFilter extends SecurityFilter implements C
} catch (InvalidTokenException ite) {
throw mappableSecurityException( INVALID_AUTH_ERROR );
}
- catch (ExternalSSOProviderAdminUserNotFoundExceptions eAdminUserNotFound){
+ catch (ExternalSSOProviderAdminUserNotFoundException eAdminUserNotFound){
throw mappableSecurityException(EXTERNALSSOPROVIDER_UNACTIVATED_ADMINUSER);
} catch(IndexOutOfBoundsException ioobe) {
// token is just some rubbish string
http://git-wip-us.apache.org/repos/asf/usergrid/blob/930308d7/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java b/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java
new file mode 100644
index 0000000..67f1b1e
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/exceptions/ExternalSSOProviderAdminUserNotFoundException.java
@@ -0,0 +1,11 @@
+package org.apache.usergrid.management.exceptions;
+
+/**
+ * Created by ayeshadastagiri on 7/8/16.
+ */
+public class ExternalSSOProviderAdminUserNotFoundException extends ManagementException {
+
+ public ExternalSSOProviderAdminUserNotFoundException(){super();}
+ public ExternalSSOProviderAdminUserNotFoundException(String arg0){super(arg0);}
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/930308d7/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
index dc61b7f..2234257 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
@@ -332,10 +332,15 @@ public class TokenServiceImpl implements TokenService {
// If the token doesn't parse as a Usergrid token, see if an external provider other than Usergrid is
// enabled. If so, just validate the external token.
- if( isExternalSSOProviderEnabled() && !getExternalSSOProvider().equalsIgnoreCase("usergrid")) {
- return validateExternalToken(token, 1, getExternalSSOProvider());
- }else{
- throw e; // re-throw the error
+ try{
+ if( isExternalSSOProviderEnabled() && !getExternalSSOProvider().equalsIgnoreCase("usergrid")) {
+ return validateExternalToken(token, 1, getExternalSSOProvider());
+ }else{
+ throw new IllegalArgumentException("invalid external provider : " + getExternalSSOProvider()); // re-throw the error
+ }
+ }
+ catch (NullPointerException npe){
+ throw new IllegalArgumentException("The SSO provider in the config is empty.");
}
}
[05/50] [abbrv] usergrid git commit: Merge branch
'apigee-sso-provider' of https://github.com/apache/usergrid into
SSO2-Usergrid
Posted by mr...@apache.org.
Merge branch 'apigee-sso-provider' of https://github.com/apache/usergrid into SSO2-Usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/526748bb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/526748bb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/526748bb
Branch: refs/heads/master
Commit: 526748bba4e40c0c2eb79d816be98f0dd62b3e39
Parents: f097e35 3764234
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Mon Jun 27 13:39:54 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Mon Jun 27 13:39:54 2016 -0700
----------------------------------------------------------------------
.../apache/usergrid/security/sso/ApigeeSSO2Provider.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[23/50] [abbrv] usergrid git commit: Initial checkin for Python
Utilities and SDK
Posted by mr...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/parse_importer/README.md
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/README.md b/utils/usergrid-util-python/usergrid_tools/parse_importer/README.md
new file mode 100644
index 0000000..3f75025
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/README.md
@@ -0,0 +1,90 @@
+# Data Importer for Parse.com Application Data Export
+
+## Overview
+
+This Python script uses the Usergrid Python SDK to iterate a data export from Parse.com to import it into a Usergrid instance
+
+## Usage
+
+```
+usage: parse_data_importer.py [-h] -o ORG -a APP --url URL -f FILE --tmp_dir
+ TMP_DIR [--client_id CLIENT_ID]
+ [--client_secret CLIENT_SECRET]
+
+Parse.com Data Importer for Usergrid
+
+optional arguments:
+ -h, --help show this help message and exit
+ -o ORG, --org ORG Name of the org to import data into
+ -a APP, --app APP Name of the app to import data into
+ --url URL The URL of the Usergrid Instance to import data into
+ -f FILE, --file FILE Full or relative path of the data file to import
+ --tmp_dir TMP_DIR Directory where data file will be unzipped
+ --client_id CLIENT_ID
+ The Client ID for using OAuth Tokens - necessary if
+ app is secured
+ --client_secret CLIENT_SECRET
+ The Client Secret for using OAuth Tokens - necessary
+ if app is secured
+```
+
+## Features
+
+Support for:
+* Roles -> Users
+* Roles -> Roles
+* Custom entities
+* Joins implemented as Graph Edges with the name of 'joins' - in both directions
+* Pointers implemented as Graph Edges with the name of 'pointers' - in both directions on an object
+
+No Support for:
+* Products - In-App Purchases
+* Installations - Will map to 'Devices' at some point - important for Push Notifications perhaps
+* Binary Assets (Images) - Work in Progress to complete
+
+## Graph Edges in Usergrid
+
+Usergrid is a Graph Datastore and implements the concept of a Graph Edge in the form of a 'connection'. Pointers, when found on an object, are implemented as follows:
+
+Source Entity --[Edge Name]--> Target Entity
+
+This is represented as a URL as follows: /{source_collection}/{source_entity_id}/pointers/{optional:target_type}. A GET on this URL would return a list of entities which have this graph edge. If a `{target_type}` is specified the results will be limited to entities of that type.
+
+Examples:
+* `GET /pets/max/pointers` - get the list of entities of all entity types which have a 'pointers' edge to them from the 'pet' 'max'
+* `GET /pets/max/pointers/owners` - get the list of entities of owners which have a 'pointers' edge to them from the 'pet' 'max'
+* `GET /pets/max/pointers/owners/jeff` - get the owner 'jeff' which has a 'pointers' edge to them from the 'pet' 'max'
+
+## Pointers
+
+Parse.com has support for pointers from one object to another. For example, for a Pointer from a Pet to an Owner, the object might look as follows:
+
+```
+{
+ "fields" : "...",
+ "objectId": "A7Hdad8HD3",
+ "owner": {
+ "__type": "Pointer",
+ "className": "Owner",
+ "objectId": "QC41NHJJlU"
+ }
+}
+```
+
+
+## Joins
+Parse.com has support for the concept of a Join as well. At the moment, Joining Users and Roles is supported and an attempt has been made to support arbitrary Joins based on the format of the `_Join:users:_Role.json` file found in my exported data. The from/to types appear to be found in the filename.
+
+An example of the Join file is below:
+
+```
+{ "results": [
+ {
+ "owningId": "lxhMWzbeXa",
+ "relatedId": "MCU2Cv9nuk"
+ }
+] }
+```
+
+
+Joins are implemented as Graph Edges with the name of 'joins' - in both directions from the objects where the Join was found
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
new file mode 100644
index 0000000..3a2d864
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
@@ -0,0 +1,385 @@
+import json
+import logging
+from logging.handlers import RotatingFileHandler
+import os
+from os import listdir
+import zipfile
+from os.path import isfile
+import sys
+import argparse
+import traceback
+
+from usergrid import Usergrid
+from usergrid.UsergridClient import UsergridEntity
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+logger = logging.getLogger('UsergridParseImporter')
+
+parse_id_to_uuid_map = {}
+global_connections = {}
+config = {}
+
+
+def init_logging(stdout_enabled=True):
+ root_logger = logging.getLogger()
+ log_file_name = './usergrid_parse_importer.log'
+ log_formatter = logging.Formatter(fmt='%(asctime)s | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=2048576000,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+ root_logger.setLevel(logging.INFO)
+
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
+
+ if stdout_enabled:
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+ root_logger.addHandler(stdout_logger)
+
+
+def convert_parse_entity(collection, parse_entity):
+ parse_entity['type'] = collection
+
+ if 'name' not in parse_entity and collection.lower() != 'users':
+ parse_entity['name'] = parse_entity['objectId']
+
+ connections = {}
+
+ for name, value in parse_entity.iteritems():
+ if isinstance(value, dict):
+ if value.get('__type') == 'Pointer':
+ class_name = value.get('className') if value.get('className')[0] != '_' else value.get('className')[1:]
+ connections[value.get('objectId')] = class_name
+
+ logger.info('Connection found from [%s: %s] to entity [%s: %s]' % (
+ collection, parse_entity['name'], class_name, value.get('objectId')))
+
+ return UsergridEntity(parse_entity), connections
+
+
+def build_usergrid_entity(collection, entity_uuid, data=None):
+ identifier = {'type': collection, 'uuid': entity_uuid}
+ data = {} if data is None else data
+ data.update(identifier)
+ return UsergridEntity(data)
+
+
+def load_users_and_roles(working_directory):
+ with open(os.path.join(working_directory, '_User.json'), 'r') as f:
+ users = json.load(f).get('results', [])
+ logger.info('Loaded [%s] Users' % len(users))
+
+ for i, parse_user in enumerate(users):
+ logger.info('Loading user [%s]: [%s / %s]' % (i, parse_user['username'], parse_user['objectId']))
+ usergrid_user, connections = convert_parse_entity('users', parse_user)
+ res = usergrid_user.save()
+
+ if res.ok:
+ logger.info('Saved user [%s]: [%s / %s]' % (i, parse_user['username'], parse_user['objectId']))
+
+ if 'uuid' in usergrid_user.entity_data:
+ parse_id_to_uuid_map[parse_user['objectId']] = usergrid_user.get('uuid')
+ else:
+ logger.error(
+ 'Error saving user [%s]: [%s / %s] - %s' % (i, parse_user['username'], parse_user['objectId'], res))
+
+ with open(os.path.join(working_directory, '_Role.json'), 'r') as f:
+ roles = json.load(f).get('results', [])
+ logger.info('Loaded [%s] Roles' % len(roles))
+
+ for i, parse_role in enumerate(roles):
+ logger.info('Loading role [%s]: [%s / %s]' % (i, parse_role['name'], parse_role['objectId']))
+ usergrid_role, connections = convert_parse_entity('roles', parse_role)
+ res = usergrid_role.save()
+
+ if res.ok:
+ logger.info('Saved role [%s]: [%s / %s]' % (i, parse_role['name'], parse_role['objectId']))
+
+ if 'uuid' in usergrid_role.entity_data:
+ parse_id_to_uuid_map[parse_role['objectId']] = usergrid_role.get('uuid')
+
+ else:
+ logger.error(
+ 'Error saving role [%s]: [%s / %s] - %s' % (i, parse_role['name'], parse_role['objectId'], res))
+
+ join_file = os.path.join(working_directory, '_Join:users:_Role.json')
+
+ if os.path.isfile(join_file) and os.path.getsize(join_file) > 0:
+ with open(join_file, 'r') as f:
+ users_to_roles = json.load(f).get('results', [])
+ logger.info('Loaded [%s] User->Roles' % len(users_to_roles))
+
+ for user_to_role in users_to_roles:
+ role_id = user_to_role['owningId']
+ role_uuid = parse_id_to_uuid_map.get(role_id)
+
+ target_role_id = user_to_role['relatedId']
+ target_role_uuid = parse_id_to_uuid_map.get(target_role_id)
+
+ if role_uuid is None or target_role_uuid is None:
+ logger.error('Failed on assigning role [%s] to user [%s]' % (role_uuid, target_role_uuid))
+ continue
+
+ target_role_entity = build_usergrid_entity('user', target_role_uuid)
+
+ res = Usergrid.assign_role(role_uuid, target_role_entity)
+
+ if res.ok:
+ logger.info('Assigned role [%s] to user [%s]' % (role_uuid, target_role_uuid))
+ else:
+ logger.error('Failed on assigning role [%s] to user [%s]' % (role_uuid, target_role_uuid))
+
+ else:
+ logger.info('No Users -> Roles to load')
+
+ join_file = os.path.join(working_directory, '_Join:roles:_Role.json')
+
+ if os.path.isfile(join_file) and os.path.getsize(join_file) > 0:
+ with open(join_file, 'r') as f:
+ users_to_roles = json.load(f).get('results', [])
+ logger.info('Loaded [%s] Roles->Roles' % len(users_to_roles))
+
+ for user_to_role in users_to_roles:
+ role_id = user_to_role['owningId']
+ role_uuid = parse_id_to_uuid_map.get(role_id)
+
+ target_role_id = user_to_role['relatedId']
+ target_role_uuid = parse_id_to_uuid_map.get(target_role_id)
+
+ if role_uuid is None or target_role_uuid is None:
+ logger.error('Failed on assigning role [%s] to role [%s]' % (role_uuid, target_role_uuid))
+ continue
+
+ target_role_entity = build_usergrid_entity('role', target_role_uuid)
+
+ res = Usergrid.assign_role(role_uuid, target_role_entity)
+
+ if res.ok:
+ logger.info('Assigned role [%s] to role [%s]' % (role_uuid, target_role_uuid))
+ else:
+ logger.error('Failed on assigning role [%s] to role [%s]' % (role_uuid, target_role_uuid))
+
+ else:
+ logger.info('No Roles -> Roles to load')
+
+
+def process_join_file(working_directory, join_file):
+ file_path = os.path.join(working_directory, join_file)
+
+ logger.warn('Processing Join file: %s' % file_path)
+
+ parts = join_file.split(':')
+
+ if len(parts) != 3:
+ logger.warn('Did not find expected 3 parts in JOIN filename: %s' % join_file)
+ return
+
+ related_type = parts[1]
+ owning_type = parts[2].split('.')[0]
+
+ owning_type = owning_type[1:] if owning_type[0] == '_' else owning_type
+
+ with open(file_path, 'r') as f:
+ try:
+ json_data = json.load(f)
+
+ except ValueError, e:
+ print traceback.format_exc(e)
+ logger.error('Unable to process file: %s' % file_path)
+ return
+
+ entities = json_data.get('results')
+
+ for join in entities:
+ owning_id = join.get('owningId')
+ related_id = join.get('relatedId')
+
+ owning_entity = build_usergrid_entity(owning_type, parse_id_to_uuid_map.get(owning_id))
+ related_entity = build_usergrid_entity(related_type, parse_id_to_uuid_map.get(related_id))
+
+ connect_entities(owning_entity, related_entity, 'joins')
+ connect_entities(related_entity, owning_entity, 'joins')
+
+
+def load_entities(working_directory):
+ files = [
+ f for f in listdir(working_directory)
+
+ if isfile(os.path.join(working_directory, f))
+ and os.path.getsize(os.path.join(working_directory, f)) > 0
+ and f not in ['_Join:roles:_Role.json',
+ '_Join:users:_Role.json',
+ '_User.json',
+ '_Product.json',
+ '_Installation.json',
+ '_Role.json']
+ ]
+
+ # sort to put join files last...
+ for data_file in sorted(files):
+ if data_file[0:6] == '_Join:':
+ process_join_file(working_directory, data_file)
+ continue
+
+ file_path = os.path.join(working_directory, data_file)
+ collection = data_file.split('.')[0]
+
+ if collection[0] == '_':
+ logger.warn('Found internal type: [%s]' % collection)
+ collection = collection[1:]
+
+ if collection not in global_connections:
+ global_connections[collection] = {}
+
+ with open(file_path, 'r') as f:
+
+ try:
+ json_data = json.load(f)
+
+ except ValueError, e:
+ print traceback.format_exc(e)
+ logger.error('Unable to process file: %s' % file_path)
+ continue
+
+ entities = json_data.get('results')
+
+ logger.info('Found [%s] entities of type [%s]' % (len(entities), collection))
+
+ for parse_entity in entities:
+ usergrid_entity, connections = convert_parse_entity(collection, parse_entity)
+ response = usergrid_entity.save()
+
+ global_connections[collection][usergrid_entity.get('uuid')] = connections
+
+ if response.ok:
+ logger.info('Saved Entity: %s' % parse_entity)
+ else:
+ logger.info('Error saving entity %s: %s' % (parse_entity, response))
+
+
+def connect_entities(from_entity, to_entity, connection_name):
+ connect_response = from_entity.connect(connection_name, to_entity)
+
+ if connect_response.ok:
+ logger.info('Successfully connected [%s / %s]--[%s]-->[%s / %s]' % (
+ from_entity.get('type'), from_entity.get('uuid'), connection_name, to_entity.get('type'),
+ to_entity.get('uuid')))
+ else:
+ logger.error('Unable to connect [%s / %s]--[%s]-->[%s / %s]: %s' % (
+ from_entity.get('type'), from_entity.get('uuid'), connection_name, to_entity.get('type'),
+ to_entity.get('uuid'), connect_response))
+
+
+def create_connections():
+ for from_collection, entity_map in global_connections.iteritems():
+
+ for from_entity_uuid, entity_connections in entity_map.iteritems():
+ from_entity = build_usergrid_entity(from_collection, from_entity_uuid)
+
+ for to_entity_id, to_entity_collection in entity_connections.iteritems():
+ to_entity = build_usergrid_entity(to_entity_collection, parse_id_to_uuid_map.get(to_entity_id))
+
+ connect_entities(from_entity, to_entity, 'pointers')
+ connect_entities(to_entity, from_entity, 'pointers')
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Parse.com Data Importer for Usergrid')
+
+ parser.add_argument('-o', '--org',
+ help='Name of the Usergrid Org to import data into - must already exist',
+ type=str,
+ required=True)
+
+ parser.add_argument('-a', '--app',
+ help='Name of the Usergrid Application to import data into - must already exist',
+ type=str,
+ required=True)
+
+ parser.add_argument('--url',
+ help='The URL of the Usergrid Instance',
+ type=str,
+ required=True)
+
+ parser.add_argument('-f', '--file',
+ help='Full or relative path of the data file to import',
+ required=True,
+ type=str)
+
+ parser.add_argument('--tmp_dir',
+ help='Directory where data file will be unzipped',
+ required=True,
+ type=str)
+
+ parser.add_argument('--client_id',
+ help='The Client ID for using OAuth Tokens - necessary if app is secured',
+ required=False,
+ type=str)
+
+ parser.add_argument('--client_secret',
+ help='The Client Secret for using OAuth Tokens - necessary if app is secured',
+ required=False,
+ type=str)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+def main():
+ global config
+ config = parse_args()
+
+ init_logging()
+
+ Usergrid.init(org_id=config.get('org'),
+ app_id=config.get('app'),
+ base_url=config.get('url'),
+ client_id=config.get('client_id'),
+ client_secret=config.get('client_secret'))
+
+ tmp_dir = config.get('tmp_dir')
+ file_path = config.get('file')
+
+ if not os.path.isfile(file_path):
+ logger.critical('File path specified [%s] is not a file!' % file_path)
+ logger.critical('Unable to continue')
+ exit(1)
+
+ if not os.path.isdir(tmp_dir):
+ logger.critical('Temp Directory path specified [%s] is not a directory!' % tmp_dir)
+ logger.critical('Unable to continue')
+ exit(1)
+
+ file_name = os.path.basename(file_path).split('.')[0]
+ working_directory = os.path.join(tmp_dir, file_name)
+
+ try:
+ with zipfile.ZipFile(file_path, 'r') as z:
+ logger.warn('Extracting files to directory: %s' % working_directory)
+ z.extractall(working_directory)
+ logger.warn('Extraction complete')
+
+ except Exception, e:
+ logger.critical(traceback.format_exc(e))
+ logger.critical('Extraction failed')
+ logger.critical('Unable to continue')
+ exit(1)
+
+ load_users_and_roles(working_directory)
+ load_entities(working_directory)
+ create_connections()
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/permissions/README.md
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/permissions/README.md b/utils/usergrid-util-python/usergrid_tools/permissions/README.md
new file mode 100644
index 0000000..8398de6
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/permissions/README.md
@@ -0,0 +1,3 @@
+# Usergrid Tools (in Python)
+
+THIS WAS USED TO SET THE PERMISISONS for /devices because it is different from /device in Jersey 2.0 and how we use Shiro
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
new file mode 100644
index 0000000..e859843
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
@@ -0,0 +1,146 @@
+import json
+from multiprocessing import Pool
+
+import requests
+
+# URL Templates for Usergrid
+import time
+
+org_management_app_url_template = "{api_url}/management/organizations/{org}/applications?client_id={client_id}&client_secret={client_secret}"
+org_management_url_template = "{api_url}/management/organizations/{org}/applications?client_id={client_id}&client_secret={client_secret}"
+org_url_template = "{api_url}/{org}?client_id={client_id}&client_secret={client_secret}"
+app_url_template = "{api_url}/{org}/{app}?client_id={client_id}&client_secret={client_secret}"
+collection_url_template = "{api_url}/{org}/{app}/{collection}?client_id={client_id}&client_secret={client_secret}"
+collection_query_url_template = "{api_url}/{org}/{app}/{collection}?ql={ql}&client_id={client_id}&client_secret={client_secret}&limit={limit}"
+collection_graph_url_template = "{api_url}/{org}/{app}/{collection}?client_id={client_id}&client_secret={client_secret}&limit={limit}"
+connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}?client_id={client_id}&client_secret={client_secret}"
+connecting_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/connecting/{verb}?client_id={client_id}&client_secret={client_secret}"
+connection_create_by_uuid_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}?client_id={client_id}&client_secret={client_secret}"
+connection_create_by_name_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_type}/{target_name}?client_id={client_id}&client_secret={client_secret}"
+get_entity_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}&connections=none"
+get_entity_url_with_connections_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}"
+put_entity_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}"
+permissions_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/permissions?client_id={client_id}&client_secret={client_secret}"
+
+user_credentials_url_template = "{api_url}/{org}/{app}/users/{uuid}/credentials"
+
+org = 'myOrg'
+
+config = {
+ "endpoint": {
+ "api_url": "https://host",
+ },
+ "credentials": {
+ "myOrg": {
+ "client_id": "foo-zw",
+ "client_secret": "bar"
+ }
+ }
+}
+
+api_url = config.get('endpoint').get('api_url')
+
+all_creds = config.get('credentials')
+
+creds = config.get('credentials').get(org)
+
+
+def post(**kwargs):
+ # print kwargs
+ # print "curl -X POST \"%s\" -d '%s'" % (kwargs.get('url'), kwargs.get('data'))
+ return requests.post(**kwargs)
+
+
+def build_role(name, title):
+ role = {
+ 'name': name,
+ 'roleName': name,
+ 'inactivity': 0,
+ 'title': title
+ }
+
+ return role
+
+
+def set_default_role(app):
+ print app
+ role_name = 'guest'
+ role = build_role('guest', 'Guest')
+
+ # # put default role
+ role_url = put_entity_url_template.format(org=org,
+ app=app,
+ uuid=role_name,
+ collection='roles',
+ api_url=api_url,
+ **creds)
+ print 'DELETE ' + role_url
+
+ # # r = requests.delete(role_url)
+ # #
+ # # if r.status_code != 200:
+ # # print 'ERROR ON DELETE'
+ # # print r.text
+ # #
+ # # time.sleep(3)
+ #
+ # # # put default role
+ # role_collection_url = collection_url_template.format(org=org,
+ # app=app,
+ # collection='roles',
+ # api_url=api_url,
+ # **creds)
+ # print 'POST ' + role_collection_url
+ #
+ # r = post(url=role_collection_url, data=json.dumps(role))
+ #
+ # if r.status_code != 200:
+ # print r.text
+
+ permissions_url = permissions_url_template.format(org=org,
+ limit=1000,
+ app=app,
+ collection='roles',
+ uuid=role_name,
+ api_url=api_url,
+ **creds)
+
+ r = post(url=permissions_url, data=json.dumps({'permission': 'post:/users'}))
+
+ r = post(url=permissions_url, data=json.dumps({'permission': 'put:/devices/*'}))
+ r = post(url=permissions_url, data=json.dumps({'permission': 'put,post:/devices'}))
+
+ r = post(url=permissions_url, data=json.dumps({'permission': 'put:/device/*'}))
+ r = post(url=permissions_url, data=json.dumps({'permission': 'put,post:/device'}))
+
+ if r.status_code != 200:
+ print r.text
+
+
+def list_apps():
+ apps = []
+ source_org_mgmt_url = org_management_url_template.format(org=org,
+ limit=1000,
+ api_url=api_url,
+ **creds)
+
+ r = requests.get(source_org_mgmt_url)
+
+ print r.text
+
+ data = r.json().get('data')
+
+ for app_uuid in data:
+
+ if 'self-care' in app_uuid:
+ parts = app_uuid.split('/')
+ apps.append(parts[1])
+
+ return apps
+
+
+apps = list_apps()
+
+pool = Pool(12)
+
+pool.map(set_default_role, apps)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/queue/README.md
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/README.md b/utils/usergrid-util-python/usergrid_tools/queue/README.md
new file mode 100644
index 0000000..fee63f7
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/queue/README.md
@@ -0,0 +1 @@
+dlq_requeue - used for taking messages out of one queue and putting them in another. useful for reading Deadletter messages and reprocessing them. You could also add filtering to the logic if you wanted.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py b/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
new file mode 100644
index 0000000..9f31e62
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
@@ -0,0 +1,143 @@
+from multiprocessing.pool import Pool
+import argparse
+import json
+import datetime
+import os
+import time
+import sys
+
+import boto
+from boto import sqs
+import requests
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+sqs_conn = None
+sqs_queue = None
+
+# THIS WAS USED TO TAKE MESSAGES OUT OF THE DEAD LETTER AND TEST WHETHER THEY EXISTED OR NOT
+
+def total_seconds(td):
+ return (td.microseconds + (td.seconds + td.days * 24.0 * 3600) * 10.0 ** 6) / 10.0 ** 6
+
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+def get_time_remaining(count, rate):
+ if rate == 0:
+ return 'NaN'
+
+ seconds = count * 1.0 / rate
+
+ m, s = divmod(seconds, 60)
+ h, m = divmod(m, 60)
+
+ return "%d:%02d:%02d" % (h, m, s)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Loader - Queue Monitor')
+
+ parser.add_argument('-c', '--config',
+ help='The queue to load into',
+ type=str,
+ default='4g.json')
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ print str(my_args)
+
+ return vars(my_args)
+
+
+def check_exists(sqs_message):
+ # checks whether an entity is deleted. if the entity is found then it prints an error message.
+ # this was used when there were many messages going to DLQ and the reason was because the entity had been deleted
+ try:
+ message = json.loads(sqs_message.get_body())
+ except ValueError:
+ print 'Unable to decode JSON: %s' % sqs_message.get_body()
+ return
+ try:
+ for event_name, event_data in message.iteritems():
+ entity_id_scope = event_data.get('entityIdScope')
+ app_id = entity_id_scope.get('applicationScope', {}).get('application', {}).get('uuid')
+ entity_id_scope = entity_id_scope.get('id')
+ entity_id = entity_id_scope.get('uuid')
+ entity_type = entity_id_scope.get('type')
+
+ url = 'http://localhost:8080/{app_id}/{entity_type}/{entity_id}'.format(
+ app_id=app_id,
+ entity_id=entity_id,
+ entity_type=entity_type
+ )
+
+ url = 'https://{host}/{basepath}/{app_id}/{entity_type}/{entity_id}'.format(
+ host='REPLACE',
+ basepath='REPLACE',
+ app_id=app_id,
+ entity_id=entity_id,
+ entity_type=entity_type
+ )
+
+ r = requests.get(url=url,
+ headers={
+ 'Authorization': 'Bearer XCA'
+ })
+
+ if r.status_code != 404:
+ print 'ERROR/FOUND [%s]: %s' % (r.status_code, url)
+ else:
+ print '[%s]: %s' % (r.status_code, url)
+ deleted = sqs_conn.delete_message_from_handle(sqs_queue, sqs_message.receipt_handle)
+ if not deleted:
+ print 'no delete!'
+
+ except KeyboardInterrupt, e:
+ raise e
+
+
+def main():
+ global sqs_conn, sqs_queue
+ args = parse_args()
+
+ start_time = datetime.datetime.utcnow()
+ first_start_time = start_time
+
+ print "first start: %s" % first_start_time
+
+ with open(args.get('config'), 'r') as f:
+ config = json.load(f)
+
+ sqs_config = config.get('sqs')
+
+ sqs_conn = boto.sqs.connect_to_region(**sqs_config)
+ queue_name = 'baas20sr_usea_baas20sr_usea_index_all_dead'
+ sqs_queue = sqs_conn.get_queue(queue_name)
+
+ last_size = sqs_queue.count()
+
+ print 'Last Size: ' + str(last_size)
+
+ pool = Pool(10)
+
+ keep_going = True
+
+ while keep_going:
+ sqs_messages = sqs_queue.get_messages(
+ num_messages=10,
+ visibility_timeout=10,
+ wait_time_seconds=10)
+
+ if len(sqs_messages) > 0:
+ pool.map(check_exists, sqs_messages)
+ else:
+ print 'DONE!'
+ pool.terminate()
+ keep_going = False
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py b/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
new file mode 100644
index 0000000..0b22770
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
@@ -0,0 +1,173 @@
+import argparse
+import json
+import datetime
+import os
+import time
+import sys
+import uuid
+from Queue import Empty
+
+import boto
+from boto import sqs
+from multiprocessing import Process, Queue
+
+from boto.sqs.message import RawMessage
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+
+def total_seconds(td):
+ return (td.microseconds + (td.seconds + td.days * 24.0 * 3600) * 10.0 ** 6) / 10.0 ** 6
+
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+def get_time_remaining(count, rate):
+ if rate == 0:
+ return 'NaN'
+
+ seconds = count * 1.0 / rate
+
+ m, s = divmod(seconds, 60)
+ h, m = divmod(m, 60)
+
+ return "%d:%02d:%02d" % (h, m, s)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Loader - Queue Monitor')
+
+ parser.add_argument('--readers',
+ help='The queue to load into',
+ type=int,
+ default=10)
+
+ parser.add_argument('--writers',
+ help='The queue to load into',
+ type=int,
+ default=10)
+
+ parser.add_argument('-c', '--config',
+ help='The queue to load into',
+ type=str,
+ default='%s/.usergrid/queue_monitor.json' % os.getenv("HOME"))
+
+ parser.add_argument('--source_queue_name',
+ help='The queue name to send messages to. If not specified the filename is used',
+ default='entities',
+ type=str)
+
+ parser.add_argument('--target_queue_name',
+ help='The queue name to send messages to. If not specified the filename is used',
+ default='entities',
+ type=str)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ print str(my_args)
+
+ return vars(my_args)
+
+
+class Writer(Process):
+ def __init__(self, queue_name, sqs_config, work_queue):
+ super(Writer, self).__init__()
+ self.queue_name = queue_name
+ self.sqs_config = sqs_config
+ self.work_queue = work_queue
+
+ def run(self):
+ sqs_conn = boto.sqs.connect_to_region(**self.sqs_config)
+
+ sqs_queue = sqs_conn.get_queue(self.queue_name)
+ sqs_queue.set_message_class(RawMessage)
+ counter = 0
+
+ # note that there is a better way but this way works. update would be to use the batch interface
+
+ batch = []
+
+ while True:
+ try:
+ body = self.work_queue.get(timeout=10)
+ counter += 1
+
+ if counter % 100 == 1:
+ print 'WRITER %s' % counter
+
+ batch.append((str(uuid.uuid1()), body, 0))
+
+ if len(batch) >= 10:
+ print 'WRITING BATCH'
+ sqs_queue.write_batch(batch, delay_seconds=300)
+ batch = []
+
+ except Empty:
+
+ if len(batch) > 0:
+ print 'WRITING BATCH'
+ sqs_queue.write_batch(batch, delay_seconds=300)
+ batch = []
+
+
+class Reader(Process):
+ def __init__(self, queue_name, sqs_config, work_queue):
+ super(Reader, self).__init__()
+ self.queue_name = queue_name
+ self.sqs_config = sqs_config
+ self.work_queue = work_queue
+
+ def run(self):
+
+ sqs_conn = boto.sqs.connect_to_region(**self.sqs_config)
+
+ sqs_queue = sqs_conn.get_queue(self.queue_name)
+ sqs_queue.set_message_class(RawMessage)
+
+ message_counter = 0
+
+ while True:
+
+ messages = sqs_queue.get_messages(num_messages=10)
+ print 'Read %s messages' % (len(messages))
+ for message in messages:
+ message_counter += 1
+
+ if message_counter % 100 == 1:
+ print 'READ: %s' % message_counter
+
+ body = message.get_body()
+ self.work_queue.put(body)
+
+ sqs_queue.delete_message_batch(messages)
+
+
+def main():
+ args = parse_args()
+
+ source_queue_name = args.get('source_queue_name')
+ target_queue_name = args.get('target_queue_name')
+
+ start_time = datetime.datetime.utcnow()
+ first_start_time = start_time
+
+ print "first start: %s" % first_start_time
+
+ with open(args.get('config'), 'r') as f:
+ config = json.load(f)
+
+ sqs_config = config.get('sqs')
+
+ work_queue = Queue()
+
+ readers = [Reader(source_queue_name, sqs_config, work_queue) for r in xrange(args.get('readers'))]
+ [r.start() for r in readers]
+
+ writers = [Writer(target_queue_name, sqs_config, work_queue) for r in xrange(args.get('writers'))]
+ [w.start() for w in writers]
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/queue/queue-config-sample.json
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/queue-config-sample.json b/utils/usergrid-util-python/usergrid_tools/queue/queue-config-sample.json
new file mode 100644
index 0000000..2b20d71
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/queue/queue-config-sample.json
@@ -0,0 +1,22 @@
+{
+ "ug_base_url": "<your usergrid api endpoint>",
+
+ "sqs": {
+ "region_name": "<aws region for using SQS>",
+ "aws_access_key_id": "<aws key for: creating queue, writing messages>",
+ "aws_secret_access_key": "<aws secret>"
+ },
+
+ "credential_map": {
+ "example1":{''
+ "comments": "for each org you want to load/publish entities to there should be an entry in this map with the org name as the key and the client_id and secret to use when publishing entities",
+ "client_id": "<client_id>",
+ "client_secret": "<client_secret>"
+ },
+ "example2":{
+ "comments": "for each org you want to load/publish entities to there should be an entry in this map with the org name as the key and the client_id and secret to use when publishing entities",
+ "client_id": "<client_id>",
+ "client_secret": "<client_secret>"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py b/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
new file mode 100644
index 0000000..7f30d06
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
@@ -0,0 +1,155 @@
+import argparse
+import json
+import datetime
+import os
+import time
+import sys
+
+import boto
+from boto import sqs
+from multiprocessing import Process, Queue
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+
+def total_seconds(td):
+ return (td.microseconds + (td.seconds + td.days * 24.0 * 3600) * 10.0 ** 6) / 10.0 ** 6
+
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+def get_time_remaining(count, rate):
+ if rate == 0:
+ return 'NaN'
+
+ seconds = count * 1.0 / rate
+
+ m, s = divmod(seconds, 60)
+ h, m = divmod(m, 60)
+
+ return "%d:%02d:%02d" % (h, m, s)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Loader - Queue Monitor')
+
+ parser.add_argument('-c', '--config',
+ help='The queue to load into',
+ type=str,
+ default='%s/.usergrid/queue_monitor.json' % os.getenv("HOME"))
+
+ parser.add_argument('-q', '--queue_name',
+ help='The queue name to send messages to. If not specified the filename is used',
+ default='entities',
+ type=str)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ print str(my_args)
+
+ return vars(my_args)
+
+
+class Deleter(Process):
+ def __init__(self, queue_name, sqs_config, work_queue):
+ super(Deleter, self).__init__()
+ self.queue_name = queue_name
+ self.sqs_config = sqs_config
+ self.work_queue = work_queue
+
+ def run(self):
+ sqs_conn = boto.sqs.connect_to_region(**self.sqs_config)
+
+ # queue = sqs_conn.get_queue(self.queue_name)
+
+ while True:
+ delete_me = self.work_queue.get()
+ delete_me.delete()
+ print 'foo'
+
+
+class Worker(Process):
+ def __init__(self, queue_name, sqs_config, delete_queue):
+ super(Worker, self).__init__()
+ self.queue_name = queue_name
+ self.sqs_config = sqs_config
+ self.delete_queue = delete_queue
+
+ def run(self):
+
+ sqs_conn = boto.sqs.connect_to_region(**self.sqs_config)
+
+ queue = sqs_conn.get_queue(self.queue_name)
+
+ last_size = queue.count()
+
+ print 'Starting Size: %s' % last_size
+
+ delete_counter = 0
+ message_counter = 0
+
+ while True:
+
+ messages = queue.get_messages(num_messages=10, visibility_timeout=300)
+
+ for message in messages:
+ message_counter += 1
+ body = message.get_body()
+
+ try:
+
+ msg = json.loads(body)
+
+ if 'entityDeleteEvent' in msg:
+ if msg['entityDeleteEvent']['entityIdScope']['id']['type'] == 'stock':
+
+ self.delete_queue.put(message)
+ delete_counter += 1
+
+ if delete_counter % 100 == 0:
+ print 'Deleted %s of %s' % (delete_counter, message_counter)
+ else:
+ # set it eligible to be read again
+ message.change_visibility(0)
+
+ print json.dumps(msg)
+
+ except:
+ pass
+
+
+
+
+def main():
+ args = parse_args()
+
+ queue_name = args.get('queue_name')
+
+ print 'queue_name=%s' % queue_name
+
+ start_time = datetime.datetime.utcnow()
+ first_start_time = start_time
+
+ print "first start: %s" % first_start_time
+
+ with open(args.get('config'), 'r') as f:
+ config = json.load(f)
+
+ sqs_config = config.get('sqs')
+ last_time = datetime.datetime.utcnow()
+
+ work_queue = Queue()
+
+ deleters = [Deleter(queue_name, sqs_config, work_queue) for x in xrange(100)]
+ [w.start() for w in deleters]
+
+ workers = [Worker(queue_name, sqs_config, work_queue) for x in xrange(100)]
+
+ [w.start() for w in workers]
+
+ time.sleep(60)
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py b/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
new file mode 100644
index 0000000..7edf1fc
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
@@ -0,0 +1,30 @@
+import json
+from collections import defaultdict
+
+import redis
+import time
+
+cache = redis.StrictRedis(host='localhost', port=6379, db=0)
+# cache.flushall()
+
+ecid_counter = defaultdict(int)
+counter = 0
+
+for key in cache.scan_iter(match='*visited'):
+
+ # print key
+ parts = key.split(':')
+ ecid = parts[0]
+
+ if ecid != 'd22a6f10-d3ef-47e3-bbe3-e1ccade5a241':
+ cache.delete(key)
+ ecid_counter[ecid] += 1
+ counter +=1
+
+ if counter % 100000 == 0 and counter != 0:
+ print json.dumps(ecid_counter, indent=2)
+ print 'Sleeping...'
+ time.sleep(60)
+ print 'AWAKE'
+
+print json.dumps(ecid_counter, indent=2)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py b/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
new file mode 100644
index 0000000..8957861
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
@@ -0,0 +1,15 @@
+import redis
+
+r = redis.Redis("localhost", 6379)
+for key in r.scan_iter():
+ # print '%s: %s' % (r.ttl(key), key)
+
+ if key[0:4] == 'http':
+ r.set(key, 1)
+ # print 'set value'
+
+ if r.ttl(key) > 3600 \
+ or key[0:3] in ['v3:', 'v2', 'v1'] \
+ or ':visited' in key:
+ r.delete(key)
+ print 'delete %s' % key
[44/50] [abbrv] usergrid git commit: updated author
Posted by mr...@apache.org.
updated author
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c47e6f68
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c47e6f68
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c47e6f68
Branch: refs/heads/master
Commit: c47e6f6858e0b6e1375644f954043e62418ec3f2
Parents: 3b9e3c4
Author: Jeff West <jw...@apigee.com>
Authored: Thu Jul 28 14:12:03 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Thu Jul 28 14:12:03 2016 -0700
----------------------------------------------------------------------
sdks/python/usergrid/UsergridApplication.py | 1 +
sdks/python/usergrid/UsergridClient.py | 2 ++
sdks/python/usergrid/UsergridCollection.py | 3 +++
sdks/python/usergrid/UsergridConnection.py | 2 ++
sdks/python/usergrid/UsergridError.py | 3 +++
sdks/python/usergrid/UsergridOrganization.py | 2 ++
6 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47e6f68/sdks/python/usergrid/UsergridApplication.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridApplication.py b/sdks/python/usergrid/UsergridApplication.py
index f11f54d..9c86efd 100644
--- a/sdks/python/usergrid/UsergridApplication.py
+++ b/sdks/python/usergrid/UsergridApplication.py
@@ -21,6 +21,7 @@ import logging
from usergrid import UsergridError, UsergridCollection
from usergrid.app_templates import app_url_template
+__author__ = 'Jeff.West@yahoo.com'
class UsergridApplication(object):
def __init__(self, app_id, client):
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47e6f68/sdks/python/usergrid/UsergridClient.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridClient.py b/sdks/python/usergrid/UsergridClient.py
index d204288..cedeaab 100644
--- a/sdks/python/usergrid/UsergridClient.py
+++ b/sdks/python/usergrid/UsergridClient.py
@@ -24,6 +24,8 @@ from usergrid.UsergridAuth import UsergridAppAuth
from usergrid.app_templates import get_entity_url_template, post_collection_url_template, put_entity_url_template, \
delete_entity_url_template, connect_entities_by_type_template, assign_role_url_template
+__author__ = 'Jeff.West@yahoo.com'
+
def value_error(message):
raise ValueError(message)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47e6f68/sdks/python/usergrid/UsergridCollection.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridCollection.py b/sdks/python/usergrid/UsergridCollection.py
index a3d3020..eb8863d 100644
--- a/sdks/python/usergrid/UsergridCollection.py
+++ b/sdks/python/usergrid/UsergridCollection.py
@@ -17,6 +17,9 @@
# * under the License.
# */
+__author__ = 'Jeff.West@yahoo.com'
+
+
class UsergridCollection(object):
def __init__(self, org_id, app_id, collection_name, client):
self.org_id = org_id
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47e6f68/sdks/python/usergrid/UsergridConnection.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridConnection.py b/sdks/python/usergrid/UsergridConnection.py
index 83836fd..c008b91 100644
--- a/sdks/python/usergrid/UsergridConnection.py
+++ b/sdks/python/usergrid/UsergridConnection.py
@@ -19,6 +19,8 @@
import logging
+__author__ = 'Jeff.West@yahoo.com'
+
class UsergridConnection(object):
def __init__(self, source_entity, verb, target_entity):
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47e6f68/sdks/python/usergrid/UsergridError.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridError.py b/sdks/python/usergrid/UsergridError.py
index 99e3507..3b2a4e0 100644
--- a/sdks/python/usergrid/UsergridError.py
+++ b/sdks/python/usergrid/UsergridError.py
@@ -16,3 +16,6 @@
# * specific language governing permissions and limitations
# * under the License.
# */
+
+__author__ = 'Jeff.West@yahoo.com'
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47e6f68/sdks/python/usergrid/UsergridOrganization.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridOrganization.py b/sdks/python/usergrid/UsergridOrganization.py
index 3ee88d2..14ad7a5 100644
--- a/sdks/python/usergrid/UsergridOrganization.py
+++ b/sdks/python/usergrid/UsergridOrganization.py
@@ -19,6 +19,8 @@
from usergrid import UsergridApplication
+__author__ = 'Jeff.West@yahoo.com'
+
class UsergridOrganization(object):
def __init__(self, org_id, client):
[46/50] [abbrv] usergrid git commit: added sample URLs for retrieving
data
Posted by mr...@apache.org.
added sample URLs for retrieving data
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/be18bc18
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/be18bc18
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/be18bc18
Branch: refs/heads/master
Commit: be18bc18dfb70c438690ce9180cc35bd2f84e9dd
Parents: ac9cfff
Author: Jeff West <jw...@apigee.com>
Authored: Thu Jul 28 15:09:27 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Thu Jul 28 15:09:27 2016 -0700
----------------------------------------------------------------------
.../samples/beacon-event-example.py | 24 ++++++++++++++++++++
1 file changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be18bc18/utils/usergrid-util-python/samples/beacon-event-example.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/beacon-event-example.py b/utils/usergrid-util-python/samples/beacon-event-example.py
index 8e8bf02..e1c3efe 100644
--- a/utils/usergrid-util-python/samples/beacon-event-example.py
+++ b/utils/usergrid-util-python/samples/beacon-event-example.py
@@ -18,6 +18,30 @@
# */
# URL Templates for Usergrid
+#
+# Get all events for a user:
+# https://usergrid.net/beacon-sample/event-example/users/jeff/events
+#
+# Get only enterStore events:
+# https://usergrid.net/beacon-sample/event-example/users/jeff/events?ql=select * where eventtype=\u2018enterStore'
+#
+# Get/filter beacon events for a user:
+# https://usergrid.net/beacon-sample/event-example/users/jeff/events?ql=select * where eventtype=\u2018beacon'
+#
+# Get latest beacon event for user:
+# https://usergrid.net/beacon-sample/event-example/users/jeff/events?ql=select * where eventtype=\u2018beacon\u2019&limit=1
+#
+# Beacon events for store:
+# https://usergrid.net/beacon-sample/event-example/users/jeff/events?ql=select * where eventtype=\u2018beacon'
+#
+# All events for store:
+# https://usergrid.net/beacon-sample/event-example/stores/store_123/events
+#
+# All events for a beacon:
+# https://usergrid.net/beacon-sample/event-example/beacons/store_456-b2/events
+#
+# Get Users who passed a specific beacon:
+# https://usergrid.net/beacon-sample/event-example/beacons/3fd4fccb-d43b-11e5-978a-123320acb31f/events;ql=select%20* where profile=1/connecting/events/users
__author__ = 'Jeff.West@yahoo.com'
[11/50] [abbrv] usergrid git commit: Handling Exception - when admin
user is not found in UG as it needs to be activated via external provider.
Posted by mr...@apache.org.
Handling Exception - when admin user is not found in UG as it needs to be activated via external provider.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b583207a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b583207a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b583207a
Branch: refs/heads/master
Commit: b583207ab9ce373300af0e19ea9a04014461a431
Parents: 3ba4e59
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Fri Jul 8 08:34:03 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Fri Jul 8 08:34:03 2016 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/rest/exceptions/AuthErrorInfo.java | 1 +
.../shiro/filters/OAuth2AccessTokenSecurityFilter.java | 6 +++++-
.../org/apache/usergrid/security/sso/ApigeeSSO2Provider.java | 3 ++-
.../usergrid/security/tokens/cassandra/TokenServiceImpl.java | 7 ++++++-
4 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b583207a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java
index 5aff66d..8b7b969 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/AuthErrorInfo.java
@@ -43,6 +43,7 @@ public enum AuthErrorInfo {
INVALID_USERNAME_OR_PASSWORD_ERROR( "auth_invalid_username_or_password",
"Unable to authenticate due to username or password being incorrect" ), //
UNVERIFIED_OAUTH_ERROR( "auth_unverified_oath", "Unable to authenticate OAuth credentials" ), //
+ EXTERNALSSOPROVIDER_UNACTIVATED_ADMINUSER("externalssoprovider_unactivated_adminuser","Admin user needs to be activated via the external provider"),
NO_DOMAIN_ERROR( "auth_no_application", "Unable to authenticate due to application not found" ), //
NOT_DOMAIN_OWNER_ERROR( "auth_not_application_owner", "" ), //
EXPIRED_ACCESS_TOKEN_ERROR( "expired_token", "Unable to authenticate due to expired access token" ), //
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b583207a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
index ca040e8..4132dd3 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
@@ -27,6 +27,7 @@ import org.apache.shiro.subject.Subject;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.management.exceptions.ExternalSSOProviderAdminUserNotFoundExceptions;
import org.apache.usergrid.management.exceptions.ManagementException;
import org.apache.usergrid.security.AuthPrincipalInfo;
import org.apache.usergrid.security.AuthPrincipalType;
@@ -108,7 +109,10 @@ public class OAuth2AccessTokenSecurityFilter extends SecurityFilter implements C
throw mappableSecurityException( EXPIRED_ACCESS_TOKEN_ERROR );
} catch (InvalidTokenException ite) {
throw mappableSecurityException( INVALID_AUTH_ERROR );
- } catch (IndexOutOfBoundsException ioobe) {
+ }
+ catch (ExternalSSOProviderAdminUserNotFoundExceptions eAdminUserNotFound){
+ throw mappableSecurityException(EXTERNALSSOPROVIDER_UNACTIVATED_ADMINUSER);
+ } catch(IndexOutOfBoundsException ioobe) {
// token is just some rubbish string
throw mappableSecurityException( BAD_ACCESS_TOKEN_ERROR );
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b583207a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
index de10591..9871cc7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
@@ -20,6 +20,7 @@ import io.jsonwebtoken.*;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.management.exceptions.ExternalSSOProviderAdminUserNotFoundExceptions;
import org.apache.usergrid.security.AuthPrincipalInfo;
import org.apache.usergrid.security.AuthPrincipalType;
import org.apache.usergrid.security.tokens.TokenInfo;
@@ -81,7 +82,7 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
UserInfo userInfo = validateAndReturnUserInfo(token, ttl);
if(userInfo == null){
- throw new RuntimeException("Unable to load user from token: "+token);
+ throw new ExternalSSOProviderAdminUserNotFoundExceptions("Unable to load user from token: "+token);
}
return new TokenInfo(UUIDUtils.newTimeUUID(), "access", 1, 1, 1, ttl,
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b583207a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
index 14d685c..dc61b7f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
@@ -794,7 +794,12 @@ public class TokenServiceImpl implements TokenService {
}
private String getExternalSSOProvider(){
- return properties.getProperty(USERGRID_EXTERNAL_PROVIDER);
+ try {
+ return properties.getProperty(USERGRID_EXTERNAL_PROVIDER);
+ }
+ catch(NullPointerException e ){
+ throw new IllegalArgumentException("External SSO provider is enabled but the provider name is empty");
+ }
}
/**
[31/50] [abbrv] usergrid git commit: Merge branch 'master' into
apigee-sso-provider
Posted by mr...@apache.org.
Merge branch 'master' into apigee-sso-provider
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d6941b7c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d6941b7c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d6941b7c
Branch: refs/heads/master
Commit: d6941b7c6037657182cfc81b3df39f99f54978b1
Parents: a22dc92 92554a4
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Jul 26 14:17:22 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Jul 26 14:17:22 2016 -0700
----------------------------------------------------------------------
docs/data-storage/collections.md | 144 ++++
docs/orgs-and-apps/application.md | 104 +++
sdks/java/pom.xml | 2 +-
.../main/resources/usergrid-default.properties | 44 +-
.../src/test/resources/usergrid-test.properties | 20 +-
stack/core/pom.xml | 819 ++++++++++---------
.../usergrid/corepersistence/CoreModule.java | 67 +-
.../corepersistence/CpEntityManager.java | 399 +++++----
.../corepersistence/CpEntityManagerFactory.java | 64 +-
.../corepersistence/CpRelationManager.java | 104 +--
.../corepersistence/EntityManagerFig.java | 9 +
.../asyncevents/EventBuilderImpl.java | 7 +-
.../index/CollectionSettings.java | 47 ++
.../index/CollectionSettingsCache.java | 59 ++
.../index/CollectionSettingsCacheFig.java | 39 +
.../index/CollectionSettingsFactory.java | 73 ++
.../index/CollectionSettingsImpl.java | 92 +++
.../index/CollectionSettingsScope.java | 27 +
.../index/CollectionSettingsScopeImpl.java | 73 ++
.../corepersistence/index/IndexSchemaCache.java | 50 --
.../index/IndexSchemaCacheFactory.java | 44 -
.../index/IndexSchemaCacheFig.java | 39 -
.../index/IndexSchemaCacheImpl.java | 120 ---
.../corepersistence/index/IndexServiceImpl.java | 72 +-
.../index/ReIndexServiceImpl.java | 41 +-
.../service/ApplicationServiceImpl.java | 27 +-
.../usergrid/persistence/EntityManager.java | 26 +-
.../apache/usergrid/persistence/PathQuery.java | 3 +-
.../usergrid/persistence/RelationManager.java | 2 +
.../index/AsyncIndexServiceTest.java | 2 +-
.../corepersistence/index/IndexServiceTest.java | 88 +-
.../resources/usergrid-custom-test.properties | 13 +
stack/corepersistence/actorsystem/pom.xml | 106 +++
.../persistence/actorsystem/ActorSystemFig.java | 83 ++
.../actorsystem/ActorSystemManager.java | 79 ++
.../actorsystem/ActorSystemManagerImpl.java | 447 ++++++++++
.../actorsystem/ActorSystemModule.java | 34 +
.../persistence/actorsystem/ClientActor.java | 205 +++++
.../actorsystem/GuiceActorProducer.java | 46 ++
.../persistence/actorsystem/RouterProducer.java | 53 ++
.../src/main/resources/application.conf | 50 ++
.../actorsystem/ActorServiceServiceTest.java | 68 ++
.../usergrid/persistence/cache/CacheScope.java | 24 +
stack/corepersistence/collection/pom.xml | 120 ++-
.../collection/EntityCollectionManager.java | 8 +-
.../EntityCollectionManagerFactory.java | 4 +-
.../persistence/collection/FieldSet.java | 12 +
.../exception/WriteUniqueVerifyException.java | 2 +-
.../collection/guice/CollectionModule.java | 9 +
.../EntityCollectionManagerFactoryImpl.java | 166 ++--
.../impl/EntityCollectionManagerImpl.java | 180 ++--
.../mvcc/stage/CollectionIoEvent.java | 14 +-
.../mvcc/stage/delete/MarkCommit.java | 36 +-
.../mvcc/stage/write/WriteCommit.java | 82 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 142 +++-
.../serialization/SerializationFig.java | 4 -
.../UniqueValueSerializationStrategy.java | 25 +-
.../serialization/impl/MutableFieldSet.java | 12 +
.../impl/UniqueFieldRowKeySerializer.java | 3 +-
.../UniqueValueSerializationStrategyImpl.java | 138 +++-
...iqueValueSerializationStrategyProxyImpl.java | 21 +-
.../uniquevalues/ReservationCache.java | 87 ++
.../uniquevalues/ReservationCacheActor.java | 87 ++
.../uniquevalues/UniqueValueActor.java | 282 +++++++
.../uniquevalues/UniqueValueException.java | 33 +
.../uniquevalues/UniqueValuesFig.java | 67 ++
.../uniquevalues/UniqueValuesRouter.java | 70 ++
.../uniquevalues/UniqueValuesService.java | 74 ++
.../uniquevalues/UniqueValuesServiceImpl.java | 369 +++++++++
.../uniquevalues/UniqueValuesTable.java | 42 +
.../uniquevalues/UniqueValuesTableImpl.java | 94 +++
.../collection/AbstractUniqueValueTest.java | 46 ++
.../collection/EntityCollectionManagerIT.java | 171 ++--
.../EntityCollectionManagerStressTest.java | 21 +-
.../collection/guice/TestCollectionModule.java | 17 +
.../mvcc/stage/delete/MarkCommitTest.java | 10 +-
.../mvcc/stage/write/WriteCommitTest.java | 8 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 121 ++-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 55 +-
...niqueValueSerializationStrategyImplTest.java | 26 +-
.../UniqueValuesServiceDeleteTest.java | 149 ++++
.../uniquevalues/UniqueValuesServiceTest.java | 185 +++++
.../src/test/resources/usergrid-CHOP.properties | 18 +
.../src/test/resources/usergrid-UNIT.properties | 19 +
.../src/test/resources/usergrid.properties | 32 +
stack/corepersistence/common/pom.xml | 23 +-
.../core/guice/SettingsValidationCluster.java | 18 +
.../core/guice/MigrationManagerRule.java | 18 +
.../guice/SettingsValidationClusterTest.java | 19 +
.../src/test/resources/usergrid-UNIT.properties | 19 +
.../src/test/resources/usergrid.properties | 19 +
.../impl/shard/DirectedEdgeMeta.java | 8 +-
stack/corepersistence/model/pom.xml | 16 +
.../persistence/model/entity/EntityMap.java | 18 +
.../persistence/model/field/ArrayField.java | 7 +
.../persistence/model/field/ByteArrayField.java | 9 +
.../persistence/model/field/DistanceField.java | 9 +
.../model/field/EntityObjectField.java | 9 +
.../persistence/model/field/FieldTypeName.java | 3 -
.../persistence/model/field/ListField.java | 7 +
.../persistence/model/field/LocationField.java | 7 +
.../persistence/model/field/NullField.java | 7 +
.../persistence/model/field/SetField.java | 7 +
.../model/field/value/EntityObject.java | 2 +-
.../persistence/model/util/EntityUtils.java | 18 +
.../persistence/model/util/UUIDGenerator.java | 18 +
.../persistence/model/field/EntityTest.java | 18 +
.../model/util/UUIDGeneratorTest.java | 18 +
stack/corepersistence/pom.xml | 72 ++
stack/corepersistence/queryindex/pom.xml | 18 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 2 +
.../index/impl/EsEntityIndexImpl.java | 94 +--
.../persistence/index/impl/EntityIndexTest.java | 12 +-
stack/pom.xml | 10 +-
stack/query-validator/pom.xml | 32 +-
.../query/validator/AbstractQueryIT.java | 23 +-
.../usergrid/query/validator/QueryITSuite.java | 12 +-
.../query/validator/users/UserQueryIT.java | 54 +-
.../test/resources/usergrid-test-context.xml | 5 +-
.../usergrid/rest/AbstractContextResource.java | 1 +
.../apache/usergrid/rest/ShutdownListener.java | 11 +
.../rest/applications/CollectionResource.java | 97 +--
.../rest/applications/ServiceResource.java | 8 +-
...ApplicationAlreadyExistsExceptionMapper.java | 32 +
.../java/org/apache/usergrid/rest/ITSetup.java | 56 +-
.../apache/usergrid/rest/NotificationsIT.java | 6 +-
.../org/apache/usergrid/rest/UniqueCatsIT.java | 233 ++++++
.../apache/usergrid/rest/UniqueValuesIT.java | 224 +++++
.../rest/UniqueValuesPerformanceIT.java | 165 ++++
.../rest/applications/ApplicationDeleteIT.java | 115 ++-
.../applications/ApplicationResourceIT.java | 2 +-
.../collection/CollectionsResourceIT.java | 251 +++---
.../collection/users/PermissionsResourceIT.java | 4 +-
.../rest/management/OrganizationsIT.java | 3 -
.../resource/endpoints/mgmt/OrgResource.java | 5 -
.../resources/corepersistence-UNIT.properties | 2 +
.../resources/usergrid-custom-test.properties | 19 +-
.../resources/usergrid-rest-deploy-context.xml | 33 +-
stack/services/pom.xml | 6 +
.../services/AbstractCollectionService.java | 71 +-
.../services/AbstractConnectionsService.java | 9 +-
.../usergrid/services/AbstractService.java | 53 +-
.../applications/ApplicationsService.java | 4 +-
.../resources/usergrid-services-context.xml | 2 +-
.../usergrid/services/ServiceInvocationIT.java | 28 +-
.../usergrid/tools/UniqueValueScanner.java | 124 +--
146 files changed, 7172 insertions(+), 1999 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d6941b7c/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d6941b7c/stack/rest/src/main/java/org/apache/usergrid/rest/AbstractContextResource.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d6941b7c/stack/services/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d6941b7c/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
[24/50] [abbrv] usergrid git commit: Initial checkin for Python
Utilities and SDK
Posted by mr...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
new file mode 100644
index 0000000..30ecc26
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
@@ -0,0 +1,2168 @@
+import os
+import uuid
+from Queue import Empty
+import argparse
+import json
+import logging
+import sys
+from multiprocessing import Queue, Process
+from sets import Set
+
+import time_uuid
+
+import datetime
+from cloghandler import ConcurrentRotatingFileHandler
+import requests
+import traceback
+import redis
+import time
+from sys import platform as _platform
+
+import signal
+
+from requests.auth import HTTPBasicAuth
+from usergrid import UsergridQueryIterator
+import urllib3
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+ECID = str(uuid.uuid1())
+key_version = 'v4'
+
+logger = logging.getLogger('GraphMigrator')
+worker_logger = logging.getLogger('Worker')
+collection_worker_logger = logging.getLogger('CollectionWorker')
+error_logger = logging.getLogger('ErrorLogger')
+audit_logger = logging.getLogger('AuditLogger')
+status_logger = logging.getLogger('StatusLogger')
+
+urllib3.disable_warnings()
+
+DEFAULT_CREATE_APPS = False
+DEFAULT_RETRY_SLEEP = 10
+DEFAULT_PROCESSING_SLEEP = 1
+
+queue = Queue()
+QSIZE_OK = False
+
+try:
+ queue.qsize()
+ QSIZE_OK = True
+except:
+ pass
+
+session_source = requests.Session()
+session_target = requests.Session()
+
+cache = None
+
+
+def total_seconds(td):
+ return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10 ** 6
+
+
+def init_logging(stdout_enabled=True):
+ root_logger = logging.getLogger()
+ root_logger.setLevel(logging.getLevelName(config.get('log_level', 'INFO')))
+
+ # root_logger.setLevel(logging.WARN)
+
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.ERROR)
+ logging.getLogger('boto').setLevel(logging.ERROR)
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+
+ log_formatter = logging.Formatter(
+ fmt='%(asctime)s | ' + ECID + ' | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ root_logger.addHandler(stdout_logger)
+
+ if stdout_enabled:
+ stdout_logger.setLevel(logging.getLevelName(config.get('log_level', 'INFO')))
+
+ # base log file
+
+ log_file_name = os.path.join(config.get('log_dir'),
+ '%s-%s-%s-migrator.log' % (config.get('org'), config.get('migrate'), ECID))
+
+ # ConcurrentRotatingFileHandler
+ rotating_file = ConcurrentRotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=404857600,
+ backupCount=0)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+ error_log_file_name = os.path.join(config.get('log_dir'), '%s-%s-%s-migrator-errors.log' % (
+ config.get('org'), config.get('migrate'), ECID))
+
+ error_rotating_file = ConcurrentRotatingFileHandler(filename=error_log_file_name,
+ mode='a',
+ maxBytes=404857600,
+ backupCount=0)
+ error_rotating_file.setFormatter(log_formatter)
+ error_rotating_file.setLevel(logging.ERROR)
+
+ root_logger.addHandler(error_rotating_file)
+
+
+entity_name_map = {
+ 'users': 'username'
+}
+
+config = {}
+
+# URL Templates for Usergrid
+org_management_app_url_template = "{api_url}/management/organizations/{org}/applications?client_id={client_id}&client_secret={client_secret}"
+org_management_url_template = "{api_url}/management/organizations/{org}/applications?client_id={client_id}&client_secret={client_secret}"
+org_url_template = "{api_url}/{org}?client_id={client_id}&client_secret={client_secret}"
+app_url_template = "{api_url}/{org}/{app}?client_id={client_id}&client_secret={client_secret}"
+collection_url_template = "{api_url}/{org}/{app}/{collection}?client_id={client_id}&client_secret={client_secret}"
+collection_query_url_template = "{api_url}/{org}/{app}/{collection}?ql={ql}&client_id={client_id}&client_secret={client_secret}&limit={limit}"
+collection_graph_url_template = "{api_url}/{org}/{app}/{collection}?client_id={client_id}&client_secret={client_secret}&limit={limit}"
+connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}?client_id={client_id}&client_secret={client_secret}"
+connecting_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/connecting/{verb}?client_id={client_id}&client_secret={client_secret}"
+connection_create_by_uuid_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}?client_id={client_id}&client_secret={client_secret}"
+connection_create_by_name_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_type}/{target_name}?client_id={client_id}&client_secret={client_secret}"
+
+connection_create_by_pairs_url_template = "{api_url}/{org}/{app}/{source_type_id}/{verb}/{target_type_id}?client_id={client_id}&client_secret={client_secret}"
+
+get_entity_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}&connections=none"
+get_entity_url_with_connections_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}"
+put_entity_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}"
+permissions_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/permissions?client_id={client_id}&client_secret={client_secret}"
+
+user_credentials_url_template = "{api_url}/{org}/{app}/users/{uuid}/credentials"
+
+ignore_collections = ['activities', 'queues', 'events', 'notifications']
+
+
+class StatusListener(Process):
+ def __init__(self, status_queue, worker_queue):
+ super(StatusListener, self).__init__()
+ self.status_queue = status_queue
+ self.worker_queue = worker_queue
+
+ def run(self):
+ keep_going = True
+
+ org_results = {
+ 'name': config.get('org'),
+ 'apps': {},
+ }
+
+ empty_count = 0
+
+ status_file_name = os.path.join(config.get('log_dir'),
+ '%s-%s-%s-status.json' % (config.get('org'), config.get('migrate'), ECID))
+
+ while keep_going:
+
+ try:
+ app, collection, status_map = self.status_queue.get(timeout=60)
+ status_logger.info('Received status update for app/collection: [%s / %s]' % (app, collection))
+ empty_count = 0
+ org_results['summary'] = {
+ 'max_created': -1,
+ 'max_modified': -1,
+ 'min_created': 1584946416000,
+ 'min_modified': 1584946416000,
+ 'count': 0,
+ 'bytes': 0
+ }
+
+ if app not in org_results['apps']:
+ org_results['apps'][app] = {
+ 'collections': {}
+ }
+
+ org_results['apps'][app]['collections'].update(status_map)
+
+ try:
+ for app, app_data in org_results['apps'].iteritems():
+ app_data['summary'] = {
+ 'max_created': -1,
+ 'max_modified': -1,
+ 'min_created': 1584946416000,
+ 'min_modified': 1584946416000,
+ 'count': 0,
+ 'bytes': 0
+ }
+
+ if 'collections' in app_data:
+ for collection, collection_data in app_data['collections'].iteritems():
+
+ app_data['summary']['count'] += collection_data['count']
+ app_data['summary']['bytes'] += collection_data['bytes']
+
+ org_results['summary']['count'] += collection_data['count']
+ org_results['summary']['bytes'] += collection_data['bytes']
+
+ # APP
+ if collection_data.get('max_modified') > app_data['summary']['max_modified']:
+ app_data['summary']['max_modified'] = collection_data.get('max_modified')
+
+ if collection_data.get('min_modified') < app_data['summary']['min_modified']:
+ app_data['summary']['min_modified'] = collection_data.get('min_modified')
+
+ if collection_data.get('max_created') > app_data['summary']['max_created']:
+ app_data['summary']['max_created'] = collection_data.get('max_created')
+
+ if collection_data.get('min_created') < app_data['summary']['min_created']:
+ app_data['summary']['min_created'] = collection_data.get('min_created')
+
+ # ORG
+ if collection_data.get('max_modified') > org_results['summary']['max_modified']:
+ org_results['summary']['max_modified'] = collection_data.get('max_modified')
+
+ if collection_data.get('min_modified') < org_results['summary']['min_modified']:
+ org_results['summary']['min_modified'] = collection_data.get('min_modified')
+
+ if collection_data.get('max_created') > org_results['summary']['max_created']:
+ org_results['summary']['max_created'] = collection_data.get('max_created')
+
+ if collection_data.get('min_created') < org_results['summary']['min_created']:
+ org_results['summary']['min_created'] = collection_data.get('min_created')
+
+ if QSIZE_OK:
+ status_logger.warn('CURRENT Queue Depth: %s' % self.worker_queue.qsize())
+
+ status_logger.warn('UPDATED status of org processed: %s' % json.dumps(org_results))
+
+ try:
+ logger.info('Writing status to file: %s' % status_file_name)
+
+ with open(status_file_name, 'w') as f:
+ json.dump(org_results, f, indent=2)
+ except:
+ print traceback.format_exc()
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except:
+ print traceback.format_exc()
+
+ except KeyboardInterrupt, e:
+ status_logger.warn('FINAL status of org processed: %s' % json.dumps(org_results))
+ raise e
+
+ except Empty:
+ if QSIZE_OK:
+ status_logger.warn('CURRENT Queue Depth: %s' % self.worker_queue.qsize())
+
+ status_logger.warn('CURRENT status of org processed: %s' % json.dumps(org_results))
+
+ status_logger.warning('EMPTY! Count=%s' % empty_count)
+
+ empty_count += 1
+
+ if empty_count >= 120:
+ keep_going = False
+
+ except:
+ print traceback.format_exc()
+
+ logger.warn('FINAL status of org processed: %s' % json.dumps(org_results))
+
+ try:
+ logger.info('Writing final status to file: %s' % status_file_name)
+ with open(status_file_name, 'w') as f:
+ json.dump(org_results, f, indent=2)
+ except:
+ print traceback.format_exc()
+
+
+class EntityWorker(Process):
+ def __init__(self, queue, handler_function):
+ super(EntityWorker, self).__init__()
+
+ worker_logger.debug('Creating worker!')
+ self.queue = queue
+ self.handler_function = handler_function
+
+ def run(self):
+
+ worker_logger.info('starting run()...')
+ keep_going = True
+
+ count_processed = 0
+ empty_count = 0
+ start_time = int(time.time())
+
+ while keep_going:
+
+ try:
+ # get an entity with the app and collection name
+ app, collection_name, entity = self.queue.get(timeout=120)
+ empty_count = 0
+
+ # if entity.get('type') == 'user':
+ # entity = confirm_user_entity(app, entity)
+
+ # the handler operation is the specified operation such as migrate_graph
+ if self.handler_function is not None:
+ try:
+ message_start_time = int(time.time())
+ processed = self.handler_function(app, collection_name, entity)
+ message_end_time = int(time.time())
+
+ if processed:
+ count_processed += 1
+
+ total_time = message_end_time - start_time
+ avg_time_per_message = total_time / count_processed
+ message_time = message_end_time - message_start_time
+
+ worker_logger.debug('Processed [%sth] entity = %s / %s / %s' % (
+ count_processed, app, collection_name, entity.get('uuid')))
+
+ if count_processed % 1000 == 1:
+ worker_logger.info(
+ 'Processed [%sth] entity = [%s / %s / %s] in [%s]s - avg time/message [%s]' % (
+ count_processed, app, collection_name, entity.get('uuid'), message_time,
+ avg_time_per_message))
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except Exception, e:
+ logger.exception('Error in EntityWorker processing message')
+ print traceback.format_exc()
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except Empty:
+ worker_logger.warning('EMPTY! Count=%s' % empty_count)
+
+ empty_count += 1
+
+ if empty_count >= 2:
+ keep_going = False
+
+ except Exception, e:
+ logger.exception('Error in EntityWorker run()')
+ print traceback.format_exc()
+
+
+class CollectionWorker(Process):
+ def __init__(self, work_queue, entity_queue, response_queue):
+ super(CollectionWorker, self).__init__()
+ collection_worker_logger.debug('Creating worker!')
+ self.work_queue = work_queue
+ self.response_queue = response_queue
+ self.entity_queue = entity_queue
+
+ def run(self):
+
+ collection_worker_logger.info('starting run()...')
+ keep_going = True
+
+ counter = 0
+ # max_created = 0
+ empty_count = 0
+ app = 'ERROR'
+ collection_name = 'NOT SET'
+ status_map = {}
+ sleep_time = 10
+
+ try:
+
+ while keep_going:
+
+ try:
+ app, collection_name = self.work_queue.get(timeout=30)
+
+ status_map = {
+ collection_name: {
+ 'iteration_started': str(datetime.datetime.now()),
+ 'max_created': -1,
+ 'max_modified': -1,
+ 'min_created': 1584946416000,
+ 'min_modified': 1584946416000,
+ 'count': 0,
+ 'bytes': 0
+ }
+ }
+
+ empty_count = 0
+
+ # added a flag for using graph vs query/index
+ if config.get('graph', False):
+ source_collection_url = collection_graph_url_template.format(org=config.get('org'),
+ app=app,
+ collection=collection_name,
+ limit=config.get('limit'),
+ **config.get('source_endpoint'))
+ else:
+ source_collection_url = collection_query_url_template.format(org=config.get('org'),
+ app=app,
+ collection=collection_name,
+ limit=config.get('limit'),
+ ql="select * %s" % config.get(
+ 'ql'),
+ **config.get('source_endpoint'))
+
+ logger.info('Iterating URL: %s' % source_collection_url)
+
+ # use the UsergridQuery from the Python SDK to iterate the collection
+ q = UsergridQueryIterator(source_collection_url,
+ page_delay=config.get('page_sleep_time'),
+ sleep_time=config.get('error_retry_sleep'))
+
+ for entity in q:
+
+ # begin entity loop
+
+ self.entity_queue.put((app, collection_name, entity))
+ counter += 1
+
+ if 'created' in entity:
+
+ try:
+ entity_created = long(entity.get('created'))
+
+ if entity_created > status_map[collection_name]['max_created']:
+ status_map[collection_name]['max_created'] = entity_created
+ status_map[collection_name]['max_created_str'] = str(
+ datetime.datetime.fromtimestamp(entity_created / 1000))
+
+ if entity_created < status_map[collection_name]['min_created']:
+ status_map[collection_name]['min_created'] = entity_created
+ status_map[collection_name]['min_created_str'] = str(
+ datetime.datetime.fromtimestamp(entity_created / 1000))
+
+ except ValueError:
+ pass
+
+ if 'modified' in entity:
+
+ try:
+ entity_modified = long(entity.get('modified'))
+
+ if entity_modified > status_map[collection_name]['max_modified']:
+ status_map[collection_name]['max_modified'] = entity_modified
+ status_map[collection_name]['max_modified_str'] = str(
+ datetime.datetime.fromtimestamp(entity_modified / 1000))
+
+ if entity_modified < status_map[collection_name]['min_modified']:
+ status_map[collection_name]['min_modified'] = entity_modified
+ status_map[collection_name]['min_modified_str'] = str(
+ datetime.datetime.fromtimestamp(entity_modified / 1000))
+
+ except ValueError:
+ pass
+
+ status_map[collection_name]['bytes'] += count_bytes(entity)
+ status_map[collection_name]['count'] += 1
+
+ if counter % 1000 == 1:
+ try:
+ collection_worker_logger.warning(
+ 'Sending stats for app/collection [%s / %s]: %s' % (
+ app, collection_name, status_map))
+
+ self.response_queue.put((app, collection_name, status_map))
+
+ if QSIZE_OK:
+ collection_worker_logger.info(
+ 'Counter=%s, collection queue depth=%s' % (
+ counter, self.work_queue.qsize()))
+ except:
+ pass
+
+ collection_worker_logger.warn(
+ 'Current status of collections processed: %s' % json.dumps(status_map))
+
+ if config.get('entity_sleep_time') > 0:
+ collection_worker_logger.debug(
+ 'sleeping for [%s]s per entity...' % (config.get('entity_sleep_time')))
+ time.sleep(config.get('entity_sleep_time'))
+ collection_worker_logger.debug(
+ 'STOPPED sleeping for [%s]s per entity...' % (config.get('entity_sleep_time')))
+
+ # end entity loop
+
+ status_map[collection_name]['iteration_finished'] = str(datetime.datetime.now())
+
+ collection_worker_logger.warning(
+ 'Collection [%s / %s / %s] loop complete! Max Created entity %s' % (
+ config.get('org'), app, collection_name, status_map[collection_name]['max_created']))
+
+ collection_worker_logger.warning(
+ 'Sending FINAL stats for app/collection [%s / %s]: %s' % (app, collection_name, status_map))
+
+ self.response_queue.put((app, collection_name, status_map))
+
+ collection_worker_logger.info('Done! Finished app/collection: %s / %s' % (app, collection_name))
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except Empty:
+ collection_worker_logger.warning('EMPTY! Count=%s' % empty_count)
+
+ empty_count += 1
+
+ if empty_count >= 2:
+ keep_going = False
+
+ except Exception, e:
+ logger.exception('Error in CollectionWorker processing collection [%s]' % collection_name)
+ print traceback.format_exc()
+
+ finally:
+ self.response_queue.put((app, collection_name, status_map))
+ collection_worker_logger.info('FINISHED!')
+
+
+def use_name_for_collection(collection_name):
+ return collection_name in config.get('use_name_for_collection', [])
+
+
+def include_edge(collection_name, edge_name):
+ include_edges = config.get('include_edge', [])
+
+ if include_edges is None:
+ include_edges = []
+
+ exclude_edges = config.get('exclude_edge', [])
+
+ if exclude_edges is None:
+ exclude_edges = []
+
+ if len(include_edges) > 0 and edge_name not in include_edges:
+ logger.debug(
+ 'Skipping edge [%s] since it is not in INCLUDED list: %s' % (edge_name, include_edges))
+ return False
+
+ if edge_name in exclude_edges:
+ logger.debug(
+ 'Skipping edge [%s] since it is in EXCLUDED list: %s' % (edge_name, exclude_edges))
+ return False
+
+ if (collection_name in ['users', 'user'] and edge_name in ['followers', 'feed', 'activities']) \
+ or (collection_name in ['receipts', 'receipt'] and edge_name in ['device', 'devices']):
+ # feed and activities are not retrievable...
+ # roles and groups will be more efficiently handled from the role/group -> user
+ # followers will be handled by 'following'
+ # do only this from user -> device
+ return False
+
+ return True
+
+
+def exclude_edge(collection_name, edge_name):
+ exclude_edges = config.get('exclude_edge', [])
+
+ if exclude_edges is None:
+ exclude_edges = []
+
+ if edge_name in exclude_edges:
+ logger.debug('Skipping edge [%s] since it is in EXCLUDED list: %s' % (edge_name, exclude_edges))
+ return True
+
+ if (collection_name in ['users', 'user'] and edge_name in ['followers', 'feed', 'activities']) \
+ or (collection_name in ['receipts', 'receipt'] and edge_name in ['device', 'devices']):
+ # feed and activities are not retrievable...
+ # roles and groups will be more efficiently handled from the role/group -> user
+ # followers will be handled by 'following'
+ # do only this from user -> device
+ return True
+
+ return False
+
+
+def confirm_user_entity(app, source_entity, attempts=0):
+ attempts += 1
+
+ source_entity_url = get_entity_url_template.format(org=config.get('org'),
+ app=app,
+ collection='users',
+ uuid=source_entity.get('username'),
+ **config.get('source_endpoint'))
+
+ if attempts >= 5:
+ logger.warning('Punting after [%s] attempts to confirm user at URL [%s], will use the source entity...' % (
+ attempts, source_entity_url))
+
+ return source_entity
+
+ r = requests.get(url=source_entity_url)
+
+ if r.status_code == 200:
+ retrieved_entity = r.json().get('entities')[0]
+
+ if retrieved_entity.get('uuid') != source_entity.get('uuid'):
+ logger.info(
+ 'UUID of Source Entity [%s] differs from uuid [%s] of retrieved entity at URL=[%s] and will be substituted' % (
+ source_entity.get('uuid'), retrieved_entity.get('uuid'), source_entity_url))
+
+ return retrieved_entity
+
+ elif 'service_resource_not_found' in r.text:
+
+ logger.warn('Unable to retrieve user at URL [%s], and will use source entity. status=[%s] response: %s...' % (
+ source_entity_url, r.status_code, r.text))
+
+ return source_entity
+
+ else:
+ logger.error('After [%s] attempts to confirm user at URL [%s], received status [%s] message: %s...' % (
+ attempts, source_entity_url, r.status_code, r.text))
+
+ time.sleep(DEFAULT_RETRY_SLEEP)
+
+ return confirm_user_entity(app, source_entity, attempts)
+
+
+def create_connection(app, collection_name, source_entity, edge_name, target_entity):
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ source_identifier = get_source_identifier(source_entity)
+ target_identifier = get_source_identifier(target_entity)
+
+ source_type_id = '%s/%s' % (source_entity.get('type'), source_identifier)
+ target_type_id = '%s/%s' % (target_entity.get('type'), target_identifier)
+
+ if source_entity.get('type') == 'user':
+ source_type_id = '%s/%s' % ('users', source_entity.get('username'))
+
+ if target_entity.get('type') == 'user':
+ if edge_name == 'users':
+ target_type_id = target_entity.get('uuid')
+ else:
+ target_type_id = '%s/%s' % ('users', target_entity.get('uuid'))
+
+ if target_entity.get('type') == 'device':
+ if edge_name == 'devices':
+ target_type_id = target_entity.get('uuid')
+ else:
+ target_type_id = '%s/%s' % ('devices', target_entity.get('uuid'))
+
+ if target_entity.get('type') == 'receipt':
+ if edge_name == 'receipts':
+ target_type_id = target_entity.get('uuid')
+ else:
+ target_type_id = '%s/%s' % ('receipts', target_entity.get('uuid'))
+
+ create_connection_url = connection_create_by_pairs_url_template.format(
+ org=target_org,
+ app=target_app,
+ source_type_id=source_type_id,
+ verb=edge_name,
+ target_type_id=target_type_id,
+ **config.get('target_endpoint'))
+
+ if not config.get('skip_cache_read', False):
+ processed = cache.get(create_connection_url)
+
+ if processed not in [None, 'None']:
+ logger.debug('Skipping visited Edge: [%s / %s / %s] --[%s]--> [%s / %s / %s]: %s ' % (
+ app, collection_name, source_identifier, edge_name, target_app, target_entity.get('type'),
+ target_entity.get('name'), create_connection_url))
+
+ return True
+
+ logger.info('Connecting entity [%s / %s / %s] --[%s]--> [%s / %s / %s]: %s ' % (
+ app, collection_name, source_identifier, edge_name, target_app, target_entity.get('type'),
+ target_entity.get('name', target_entity.get('uuid')), create_connection_url))
+
+ attempts = 0
+
+ while attempts < 5:
+ attempts += 1
+
+ r_create = session_target.post(create_connection_url)
+
+ if r_create.status_code == 200:
+
+ if not config.get('skip_cache_write', False):
+ cache.set(create_connection_url, 1)
+
+ return True
+ else:
+ if r_create.status_code >= 500:
+
+ if attempts < 5:
+ logger.warning('FAILED [%s] (will retry) to create connection at URL=[%s]: %s' % (
+ r_create.status_code, create_connection_url, r_create.text))
+ time.sleep(DEFAULT_RETRY_SLEEP)
+ else:
+ logger.critical(
+ 'FAILED [%s] (WILL NOT RETRY - max attempts) to create connection at URL=[%s]: %s' % (
+ r_create.status_code, create_connection_url, r_create.text))
+ return False
+
+ elif r_create.status_code in [401, 404]:
+
+ if config.get('repair_data', False):
+ logger.warning('FAILED [%s] (WILL attempt repair) to create connection at URL=[%s]: %s' % (
+ r_create.status_code, create_connection_url, r_create.text))
+ migrate_data(app, source_entity.get('type'), source_entity, force=True)
+ migrate_data(app, target_entity.get('type'), target_entity, force=True)
+
+ else:
+ logger.critical('FAILED [%s] (WILL NOT attempt repair) to create connection at URL=[%s]: %s' % (
+ r_create.status_code, create_connection_url, r_create.text))
+
+ else:
+ logger.warning('FAILED [%s] (will retry) to create connection at URL=[%s]: %s' % (
+ r_create.status_code, create_connection_url, r_create.text))
+
+ return False
+
+
+def process_edges(app, collection_name, source_entity, edge_name, connection_stack):
+
+ source_identifier = get_source_identifier(source_entity)
+
+ while len(connection_stack) > 0:
+
+ target_entity = connection_stack.pop()
+
+ if exclude_collection(collection_name) or exclude_collection(target_entity.get('type')):
+ logger.debug('EXCLUDING Edge (collection): [%s / %s / %s] --[%s]--> ?' % (
+ app, collection_name, source_identifier, edge_name ))
+ continue
+
+ create_connection(app, collection_name, source_entity, edge_name, target_entity)
+
+
+def migrate_out_graph_edge_type(app, collection_name, source_entity, edge_name, depth=0):
+ if not include_edge(collection_name, edge_name):
+ return True
+
+ source_uuid = source_entity.get('uuid')
+
+ key = '%s:edge:out:%s:%s' % (key_version, source_uuid, edge_name)
+
+ if not config.get('skip_cache_read', False):
+ date_visited = cache.get(key)
+
+ if date_visited not in [None, 'None']:
+ logger.info('Skipping EDGE [%s / %s --%s-->] - visited at %s' % (
+ collection_name, source_uuid, edge_name, date_visited))
+ return True
+ else:
+ cache.delete(key)
+
+ if not config.get('skip_cache_write', False):
+ cache.set(name=key, value=str(int(time.time())), ex=config.get('visit_cache_ttl', 3600 * 2))
+
+ logger.debug('Visiting EDGE [%s / %s (%s) --%s-->] at %s' % (
+ collection_name, source_uuid, get_uuid_time(source_uuid), edge_name, str(datetime.datetime.utcnow())))
+
+ response = True
+
+ source_identifier = get_source_identifier(source_entity)
+
+ count_edges = 0
+
+ logger.debug(
+ 'Processing edge type=[%s] of entity [%s / %s / %s]' % (edge_name, app, collection_name, source_identifier))
+
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ connection_query_url = connection_query_url_template.format(
+ org=config.get('org'),
+ app=app,
+ verb=edge_name,
+ collection=collection_name,
+ uuid=source_identifier,
+ limit=config.get('limit'),
+ **config.get('source_endpoint'))
+
+ connection_query = UsergridQueryIterator(connection_query_url, sleep_time=config.get('error_retry_sleep'))
+
+ connection_stack = []
+
+ for target_entity in connection_query:
+ target_connection_collection = config.get('collection_mapping', {}).get(target_entity.get('type'),
+ target_entity.get('type'))
+
+ target_ok = migrate_graph(app, target_entity.get('type'), source_entity=target_entity, depth=depth)
+
+ if not target_ok:
+ logger.critical(
+ 'Error migrating TARGET entity data for connection [%s / %s / %s] --[%s]--> [%s / %s / %s]' % (
+ app, collection_name, source_identifier, edge_name, app, target_connection_collection,
+ target_entity.get('name', target_entity.get('uuid'))))
+
+ count_edges += 1
+ connection_stack.append(target_entity)
+
+ process_edges(app, collection_name, source_entity, edge_name, connection_stack)
+
+ return response
+
+
+def get_source_identifier(source_entity):
+ entity_type = source_entity.get('type')
+
+ source_identifier = source_entity.get('uuid')
+
+ if use_name_for_collection(entity_type):
+
+ if entity_type in ['user']:
+ source_identifier = source_entity.get('username')
+ else:
+ source_identifier = source_entity.get('name')
+
+ if source_identifier is None:
+ source_identifier = source_entity.get('uuid')
+ logger.warn('Using UUID for entity [%s / %s]' % (entity_type, source_identifier))
+
+ return source_identifier
+
+
+def include_collection(collection_name):
+ if collection_name in ['events']:
+ return False
+
+ include = config.get('collection', [])
+
+ if include is not None and len(include) > 0 and collection_name not in include:
+ return False
+
+ exclude = config.get('exclude_collection', [])
+
+ if exclude is not None and collection_name in exclude:
+ return False
+
+ return True
+
+
+def exclude_collection(collection_name):
+ exclude = config.get('exclude_collection', [])
+
+ if exclude is not None and collection_name in exclude:
+ return True
+
+ return False
+
+
+def migrate_in_graph_edge_type(app, collection_name, source_entity, edge_name, depth=0):
+ source_uuid = source_entity.get('uuid')
+ key = '%s:edges:in:%s:%s' % (key_version, source_uuid, edge_name)
+
+ if not config.get('skip_cache_read', False):
+ date_visited = cache.get(key)
+
+ if date_visited not in [None, 'None']:
+ logger.info('Skipping EDGE [--%s--> %s / %s] - visited at %s' % (
+ collection_name, source_uuid, edge_name, date_visited))
+ return True
+ else:
+ cache.delete(key)
+
+ if not config.get('skip_cache_write', False):
+ cache.set(name=key, value=str(int(time.time())), ex=config.get('visit_cache_ttl', 3600 * 2))
+
+ logger.debug('Visiting EDGE [--%s--> %s / %s (%s)] at %s' % (
+ edge_name, collection_name, source_uuid, get_uuid_time(source_uuid), str(datetime.datetime.utcnow())))
+
+ source_identifier = get_source_identifier(source_entity)
+
+ if exclude_collection(collection_name):
+ logger.debug('Excluding (Collection) entity [%s / %s / %s]' % (app, collection_name, source_uuid))
+ return True
+
+ if not include_edge(collection_name, edge_name):
+ return True
+
+ logger.debug(
+ 'Processing edge type=[%s] of entity [%s / %s / %s]' % (edge_name, app, collection_name, source_identifier))
+
+ logger.debug('Processing IN edges type=[%s] of entity [ %s / %s / %s]' % (
+ edge_name, app, collection_name, source_uuid))
+
+ connecting_query_url = connecting_query_url_template.format(
+ org=config.get('org'),
+ app=app,
+ collection=collection_name,
+ uuid=source_uuid,
+ verb=edge_name,
+ limit=config.get('limit'),
+ **config.get('source_endpoint'))
+
+ connection_query = UsergridQueryIterator(connecting_query_url, sleep_time=config.get('error_retry_sleep'))
+
+ response = True
+
+ for e_connection in connection_query:
+ logger.debug('Triggering IN->OUT edge migration on entity [%s / %s / %s] ' % (
+ app, e_connection.get('type'), e_connection.get('uuid')))
+
+ response = migrate_graph(app, e_connection.get('type'), e_connection, depth) and response
+
+ return response
+
+
+def migrate_graph(app, collection_name, source_entity, depth=0):
+ depth += 1
+ source_uuid = source_entity.get('uuid')
+
+ # short circuit if the graph depth exceeds what was specified
+ if depth > config.get('graph_depth', 1):
+ logger.debug(
+ 'Reached Max Graph Depth, stopping after [%s] on [%s / %s]' % (depth, collection_name, source_uuid))
+ return True
+ else:
+ logger.debug('Processing @ Graph Depth [%s]' % depth)
+
+ if exclude_collection(collection_name):
+ logger.warn('Ignoring entity in filtered collection [%s]' % collection_name)
+ return True
+
+ key = '%s:graph:%s' % (key_version, source_uuid)
+ entity_tag = '[%s / %s / %s (%s)]' % (app, collection_name, source_uuid, get_uuid_time(source_uuid))
+
+ if not config.get('skip_cache_read', False):
+ date_visited = cache.get(key)
+
+ if date_visited not in [None, 'None']:
+ logger.debug('Skipping GRAPH %s at %s' % (entity_tag, date_visited))
+ return True
+ else:
+ cache.delete(key)
+
+ logger.info('Visiting GRAPH %s at %s' % (entity_tag, str(datetime.datetime.utcnow())))
+
+ if not config.get('skip_cache_write', False):
+ cache.set(name=key, value=str(int(time.time())), ex=config.get('visit_cache_ttl', 3600 * 2))
+
+ # first, migrate data for current node
+ response = migrate_data(app, collection_name, source_entity)
+
+ # gather the outbound edge names
+ out_edge_names = [edge_name for edge_name in source_entity.get('metadata', {}).get('collections', [])]
+ out_edge_names += [edge_name for edge_name in source_entity.get('metadata', {}).get('connections', [])]
+
+ logger.debug('Entity %s has [%s] OUT edges' % (entity_tag, len(out_edge_names)))
+
+ # migrate each outbound edge type
+ for edge_name in out_edge_names:
+
+ if not exclude_edge(collection_name, edge_name):
+ response = migrate_out_graph_edge_type(app, collection_name, source_entity, edge_name, depth) and response
+
+ if config.get('prune', False):
+ prune_edge_by_name(edge_name, app, collection_name, source_entity)
+
+ # gather the inbound edge names
+ in_edge_names = [edge_name for edge_name in source_entity.get('metadata', {}).get('connecting', [])]
+
+ logger.debug('Entity %s has [%s] IN edges' % (entity_tag, len(in_edge_names)))
+
+ # migrate each inbound edge type
+ for edge_name in in_edge_names:
+
+ if not exclude_edge(collection_name, edge_name):
+ response = migrate_in_graph_edge_type(app, collection_name, source_entity, edge_name,
+ depth) and response
+
+ return response
+
+
+def collect_entities(q):
+ response = {}
+
+ for e in q:
+ response[e.get('uuid')] = e
+
+ return response
+
+
+def prune_edge_by_name(edge_name, app, collection_name, source_entity):
+ if not include_edge(collection_name, edge_name):
+ return True
+
+ source_identifier = get_source_identifier(source_entity)
+ source_uuid = source_entity.get('uuid')
+
+ entity_tag = '[%s / %s / %s (%s)]' % (app, collection_name, source_uuid, get_uuid_time(source_uuid))
+
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ target_connection_query_url = connection_query_url_template.format(
+ org=target_org,
+ app=target_app,
+ verb=edge_name,
+ collection=target_collection,
+ uuid=source_identifier,
+ limit=config.get('limit'),
+ **config.get('target_endpoint'))
+
+ source_connection_query_url = connection_query_url_template.format(
+ org=config.get('org'),
+ app=app,
+ verb=edge_name,
+ collection=collection_name,
+ uuid=source_identifier,
+ limit=config.get('limit'),
+ **config.get('source_endpoint'))
+
+ source_connections = collect_entities(
+ UsergridQueryIterator(source_connection_query_url, sleep_time=config.get('error_retry_sleep')))
+
+ target_connections = collect_entities(
+ UsergridQueryIterator(target_connection_query_url, sleep_time=config.get('error_retry_sleep')))
+
+ delete_uuids = Set(target_connections.keys()) - Set(source_connections.keys())
+
+ if len(delete_uuids) > 0:
+ logger.info('Found [%s] edges to delete for entity %s' % (len(delete_uuids), entity_tag))
+
+ for delete_uuid in delete_uuids:
+ delete_connection_url = connection_create_by_uuid_url_template.format(
+ org=target_org,
+ app=target_app,
+ verb=edge_name,
+ collection=target_collection,
+ uuid=source_identifier,
+ target_uuid=delete_uuid,
+ **config.get('target_endpoint'))
+
+ attempts = 0
+
+ while attempts < 5:
+ attempts += 1
+
+ r = session_target.delete(delete_connection_url)
+
+ if not config.get('skip_cache_write'):
+ cache.delete(delete_connection_url)
+
+ if r.status_code == 200:
+ logger.info('Pruned edge on attempt [%s] URL=[%s]' % (attempts, delete_connection_url))
+ break
+ else:
+ logger.error('Error [%s] on attempt [%s] deleting connection at URL=[%s]: %s' % (
+ r.status_code, attempts, delete_connection_url, r.text))
+ time.sleep(DEFAULT_RETRY_SLEEP)
+
+ return True
+
+
+def prune_graph(app, collection_name, source_entity):
+ source_uuid = source_entity.get('uuid')
+ key = '%s:prune_graph:%s' % (key_version, source_uuid)
+ entity_tag = '[%s / %s / %s (%s)]' % (app, collection_name, source_uuid, get_uuid_time(source_uuid))
+
+ if not config.get('skip_cache_read', False):
+ date_visited = cache.get(key)
+
+ if date_visited not in [None, 'None']:
+ logger.debug('Skipping PRUNE %s at %s' % (entity_tag, date_visited))
+ return True
+ else:
+ cache.delete(key)
+
+ logger.debug('pruning GRAPH %s at %s' % (entity_tag, str(datetime.datetime.utcnow())))
+ if not config.get('skip_cache_write', False):
+ cache.set(name=key, value=str(int(time.time())), ex=config.get('visit_cache_ttl', 3600 * 2))
+
+ if collection_name in config.get('exclude_collection', []):
+ logger.debug('Excluding (Collection) entity %s' % entity_tag)
+ return True
+
+ out_edge_names = [edge_name for edge_name in source_entity.get('metadata', {}).get('collections', [])]
+ out_edge_names += [edge_name for edge_name in source_entity.get('metadata', {}).get('connections', [])]
+
+ for edge_name in out_edge_names:
+ prune_edge_by_name(edge_name, app, collection_name, source_entity)
+
+
+def reput(app, collection_name, source_entity, attempts=0):
+ source_identifier = source_entity.get('uuid')
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ try:
+ target_entity_url_by_name = put_entity_url_template.format(org=target_org,
+ app=target_app,
+ collection=target_collection,
+ uuid=source_identifier,
+ **config.get('target_endpoint'))
+
+ r = session_source.put(target_entity_url_by_name, data=json.dumps({}))
+ if r.status_code != 200:
+ logger.info('HTTP [%s]: %s' % (target_entity_url_by_name, r.status_code))
+ else:
+ logger.debug('HTTP [%s]: %s' % (target_entity_url_by_name, r.status_code))
+
+ except:
+ pass
+
+
+def get_uuid_time(the_uuid_string):
+ return time_uuid.TimeUUID(the_uuid_string).get_datetime()
+
+
+def migrate_permissions(app, collection_name, source_entity, attempts=0):
+ if collection_name not in ['roles', 'role', 'group', 'groups']:
+ return True
+
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ source_identifier = get_source_identifier(source_entity)
+
+ source_permissions_url = permissions_url_template.format(org=config.get('org'),
+ app=app,
+ collection=collection_name,
+ uuid=source_identifier,
+ **config.get('source_endpoint'))
+
+ r = session_source.get(source_permissions_url)
+
+ if r.status_code != 200:
+ logger.error('Unable to get permissions at URL [%s]: %s' % (source_permissions_url, r.text))
+ return False
+
+ perm_response = r.json()
+
+ perms = perm_response.get('data', [])
+
+ logger.info('Migrating [%s / %s] with permissions %s' % (collection_name, source_identifier, perms))
+
+ if len(perms) > 0:
+ target_permissions_url = permissions_url_template.format(org=target_org,
+ app=target_app,
+ collection=target_collection,
+ uuid=source_identifier,
+ **config.get('target_endpoint'))
+
+ for permission in perms:
+ data = {'permission': permission}
+
+ logger.info('Posting permission %s to %s' % (json.dumps(data), target_permissions_url))
+
+ r = session_target.post(target_permissions_url, json.dumps(data))
+
+ if r.status_code != 200:
+ logger.error(
+ 'ERROR posting permission %s to URL=[%s]: %s' % (
+ json.dumps(data), target_permissions_url, r.text))
+
+ return True
+
+
+def migrate_data(app, collection_name, source_entity, attempts=0, force=False):
+ if config.get('skip_data') and not force:
+ return True
+
+ # check the cache to see if this entity has changed
+ if not config.get('skip_cache_read', False) and not force:
+ try:
+ str_modified = cache.get(source_entity.get('uuid'))
+
+ if str_modified not in [None, 'None']:
+
+ modified = long(str_modified)
+
+ logger.debug('FOUND CACHE: %s = %s ' % (source_entity.get('uuid'), modified))
+
+ if modified <= source_entity.get('modified'):
+
+ modified_date = datetime.datetime.utcfromtimestamp(modified / 1000)
+ e_uuid = source_entity.get('uuid')
+
+ uuid_datetime = time_uuid.TimeUUID(e_uuid).get_datetime()
+
+ logger.debug('Skipping ENTITY: %s / %s / %s / %s (%s) / %s (%s)' % (
+ config.get('org'), app, collection_name, e_uuid, uuid_datetime, modified, modified_date))
+ return True
+ else:
+ logger.debug('DELETING CACHE: %s ' % (source_entity.get('uuid')))
+ cache.delete(source_entity.get('uuid'))
+ except:
+ logger.error('Error on checking cache for uuid=[%s]' % source_entity.get('uuid'))
+ logger.error(traceback.format_exc())
+
+ if exclude_collection(collection_name):
+ logger.warn('Excluding entity in filtered collection [%s]' % collection_name)
+ return True
+
+ # handle duplicate user case
+ if collection_name in ['users', 'user']:
+ source_entity = confirm_user_entity(app, source_entity)
+
+ source_identifier = get_source_identifier(source_entity)
+
+ logger.info('Visiting ENTITY data [%s / %s (%s) ] at %s' % (
+ collection_name, source_identifier, get_uuid_time(source_entity.get('uuid')), str(datetime.datetime.utcnow())))
+
+ entity_copy = source_entity.copy()
+
+ if 'metadata' in entity_copy:
+ entity_copy.pop('metadata')
+
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ try:
+ target_entity_url_by_name = put_entity_url_template.format(org=target_org,
+ app=target_app,
+ collection=target_collection,
+ uuid=source_identifier,
+ **config.get('target_endpoint'))
+
+ r = session_target.put(url=target_entity_url_by_name, data=json.dumps(entity_copy))
+
+ if attempts > 1:
+ logger.warn('Attempt [%s] to migrate entity [%s / %s] at URL [%s]' % (
+ attempts, collection_name, source_identifier, target_entity_url_by_name))
+ else:
+ logger.debug('Attempt [%s] to migrate entity [%s / %s] at URL [%s]' % (
+ attempts, collection_name, source_identifier, target_entity_url_by_name))
+
+ if r.status_code == 200:
+ # Worked => WE ARE DONE
+ logger.info(
+ 'migrate_data | success=[%s] | attempts=[%s] | entity=[%s / %s / %s] | created=[%s] | modified=[%s]' % (
+ True, attempts, config.get('org'), app, source_identifier, source_entity.get('created'),
+ source_entity.get('modified'),))
+
+ if not config.get('skip_cache_write', False):
+ logger.debug('SETTING CACHE | uuid=[%s] | modified=[%s]' % (
+ source_entity.get('uuid'), str(source_entity.get('modified'))))
+
+ cache.set(source_entity.get('uuid'), str(source_entity.get('modified')))
+
+ if collection_name in ['role', 'group', 'roles', 'groups']:
+ migrate_permissions(app, collection_name, source_entity, attempts=0)
+
+ if collection_name in ['users', 'user']:
+ migrate_user_credentials(app, collection_name, source_entity, attempts=0)
+
+ return True
+
+ else:
+ logger.error('Failure [%s] on attempt [%s] to PUT url=[%s], entity=[%s] response=[%s]' % (
+ r.status_code, attempts, target_entity_url_by_name, json.dumps(source_entity), r.text))
+
+ if attempts >= 5:
+ logger.critical(
+ 'ABORT migrate_data | success=[%s] | attempts=[%s] | created=[%s] | modified=[%s] %s / %s / %s' % (
+ True, attempts, source_entity.get('created'), source_entity.get('modified'), app,
+ collection_name, source_identifier))
+
+ return False
+
+ if r.status_code == 400:
+
+ if target_collection in ['roles', 'role']:
+ return repair_user_role(app, collection_name, source_entity)
+
+ elif target_collection in ['users', 'user']:
+ return handle_user_migration_conflict(app, collection_name, source_entity)
+
+ elif 'duplicate_unique_property_exists' in r.text:
+ logger.error(
+ 'WILL NOT RETRY (duplicate) [%s] attempts to PUT url=[%s], entity=[%s] response=[%s]' % (
+ attempts, target_entity_url_by_name, json.dumps(source_entity), r.text))
+
+ return False
+
+ elif r.status_code == 403:
+ logger.critical(
+ 'ABORT migrate_data | success=[%s] | attempts=[%s] | created=[%s] | modified=[%s] %s / %s / %s' % (
+ False, attempts, source_entity.get('created'), source_entity.get('modified'), app,
+ collection_name, source_identifier))
+ return False
+
+ except:
+ logger.error(traceback.format_exc())
+ logger.error('error in migrate_data on entity: %s' % json.dumps(source_entity))
+
+ logger.warn(
+ 'UNSUCCESSFUL migrate_data | success=[%s] | attempts=[%s] | entity=[%s / %s / %s] | created=[%s] | modified=[%s]' % (
+ True, attempts, config.get('org'), app, source_identifier, source_entity.get('created'),
+ source_entity.get('modified'),))
+
+ return migrate_data(app, collection_name, source_entity, attempts=attempts + 1)
+
+
+def handle_user_migration_conflict(app, collection_name, source_entity, attempts=0, depth=0):
+ if collection_name in ['users', 'user']:
+ return False
+
+ username = source_entity.get('username')
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ target_entity_url = get_entity_url_template.format(org=target_org,
+ app=target_app,
+ collection=target_collection,
+ uuid=username,
+ **config.get('target_endpoint'))
+
+ # There is retry build in, here is the short circuit
+ if attempts >= 5:
+ logger.critical(
+ 'Aborting after [%s] attempts to audit user [%s] at URL [%s]' % (attempts, username, target_entity_url))
+
+ return False
+
+ r = session_target.get(url=target_entity_url)
+
+ if r.status_code == 200:
+ target_entity = r.json().get('entities')[0]
+
+ if source_entity.get('created') < target_entity.get('created'):
+ return repair_user_role(app, collection_name, source_entity)
+
+ elif r.status_code / 100 == 5:
+ audit_logger.warning(
+ 'CONFLICT: handle_user_migration_conflict failed attempt [%s] GET [%s] on TARGET URL=[%s] - : %s' % (
+ attempts, r.status_code, target_entity_url, r.text))
+
+ time.sleep(DEFAULT_RETRY_SLEEP)
+
+ return handle_user_migration_conflict(app, collection_name, source_entity, attempts)
+
+ else:
+ audit_logger.error(
+ 'CONFLICT: Failed handle_user_migration_conflict attempt [%s] GET [%s] on TARGET URL=[%s] - : %s' % (
+ attempts, r.status_code, target_entity_url, r.text))
+
+ return False
+
+
+def get_best_source_entity(app, collection_name, source_entity, depth=0):
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ target_pk = 'uuid'
+
+ if target_collection in ['users', 'user']:
+ target_pk = 'username'
+ elif target_collection in ['roles', 'role']:
+ target_pk = 'name'
+
+ target_name = source_entity.get(target_pk)
+
+ # there should be no target entity now, we just need to decide which one from the source to use
+ source_entity_url_by_name = get_entity_url_template.format(org=config.get('org'),
+ app=app,
+ collection=collection_name,
+ uuid=target_name,
+ **config.get('source_endpoint'))
+
+ r_get_source_entity = session_source.get(source_entity_url_by_name)
+
+ # if we are able to get at the source by PK...
+ if r_get_source_entity.status_code == 200:
+
+ # extract the entity from the response
+ entity_from_get = r_get_source_entity.json().get('entities')[0]
+
+ return entity_from_get
+
+ elif r_get_source_entity.status_code / 100 == 4:
+ # wasn't found, get by QL and sort
+ source_entity_query_url = collection_query_url_template.format(org=config.get('org'),
+ app=app,
+ collection=collection_name,
+ ql='select * where %s=\'%s\' order by created asc' % (
+ target_pk, target_name),
+ limit=config.get('limit'),
+ **config.get('source_endpoint'))
+
+ logger.info('Attempting to determine best entity from query on URL %s' % source_entity_query_url)
+
+ q = UsergridQueryIterator(source_entity_query_url, sleep_time=config.get('error_retry_sleep'))
+
+ desired_entity = None
+
+ entity_counter = 0
+
+ for e in q:
+ entity_counter += 1
+
+ if desired_entity is None:
+ desired_entity = e
+
+ elif e.get('created') < desired_entity.get('created'):
+ desired_entity = e
+
+ if desired_entity is None:
+ logger.warn('Unable to determine best of [%s] entities from query on URL %s' % (
+ entity_counter, source_entity_query_url))
+
+ return source_entity
+
+ else:
+ return desired_entity
+
+ else:
+ return source_entity
+
+
+def repair_user_role(app, collection_name, source_entity, attempts=0, depth=0):
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ # For the users collection, there seemed to be cases where a USERNAME was created/existing with the a
+ # different UUID which caused a 'collision' - so the point is to delete the entity with the differing
+ # UUID by UUID and then do a recursive call to migrate the data - now that the collision has been cleared
+
+ target_pk = 'uuid'
+
+ if target_collection in ['users', 'user']:
+ target_pk = 'username'
+ elif target_collection in ['roles', 'role']:
+ target_pk = 'name'
+
+ target_name = source_entity.get(target_pk)
+
+ target_entity_url_by_name = get_entity_url_template.format(org=target_org,
+ app=target_app,
+ collection=target_collection,
+ uuid=target_name,
+ **config.get('target_endpoint'))
+
+ logger.warning('Repairing: Deleting name=[%s] entity at URL=[%s]' % (target_name, target_entity_url_by_name))
+
+ r = session_target.delete(target_entity_url_by_name)
+
+ if r.status_code == 200 or (r.status_code in [404, 401] and 'service_resource_not_found' in r.text):
+ logger.info('Deletion of entity at URL=[%s] was [%s]' % (target_entity_url_by_name, r.status_code))
+
+ best_source_entity = get_best_source_entity(app, collection_name, source_entity)
+
+ target_entity_url_by_uuid = get_entity_url_template.format(org=target_org,
+ app=target_app,
+ collection=target_collection,
+ uuid=best_source_entity.get('uuid'),
+ **config.get('target_endpoint'))
+
+ r = session_target.put(target_entity_url_by_uuid, data=json.dumps(best_source_entity))
+
+ if r.status_code == 200:
+ logger.info('Successfully repaired user at URL=[%s]' % target_entity_url_by_uuid)
+ return True
+
+ else:
+ logger.critical('Failed to PUT [%s] the desired entity at URL=[%s]: %s' % (
+ r.status_code, target_entity_url_by_name, r.text))
+ return False
+
+ else:
+ # log an error and keep going if we cannot delete the entity at the specified URL. Unlikely, but if so
+ # then this entity is borked
+ logger.critical(
+ 'Deletion of entity at URL=[%s] FAILED [%s]: %s' % (target_entity_url_by_name, r.status_code, r.text))
+ return False
+
+
+def get_target_mapping(app, collection_name):
+ target_org = config.get('org_mapping', {}).get(config.get('org'), config.get('org'))
+ target_app = config.get('app_mapping', {}).get(app, app)
+ target_collection = config.get('collection_mapping', {}).get(collection_name, collection_name)
+ return target_app, target_collection, target_org
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Org/App Migrator')
+
+ parser.add_argument('--log_dir',
+ help='path to the place where logs will be written',
+ default='./',
+ type=str,
+ required=False)
+
+ parser.add_argument('--log_level',
+ help='log level - DEBUG, INFO, WARN, ERROR, CRITICAL',
+ default='INFO',
+ type=str,
+ required=False)
+
+ parser.add_argument('-o', '--org',
+ help='Name of the org to migrate',
+ type=str,
+ required=True)
+
+ parser.add_argument('-a', '--app',
+ help='Name of one or more apps to include, specify none to include all apps',
+ required=False,
+ action='append')
+
+ parser.add_argument('-e', '--include_edge',
+ help='Name of one or more edges/connection types to INCLUDE, specify none to include all edges',
+ required=False,
+ action='append')
+
+ parser.add_argument('--exclude_edge',
+ help='Name of one or more edges/connection types to EXCLUDE, specify none to include all edges',
+ required=False,
+ action='append')
+
+ parser.add_argument('--exclude_collection',
+ help='Name of one or more collections to EXCLUDE, specify none to include all collections',
+ required=False,
+ action='append')
+
+ parser.add_argument('-c', '--collection',
+ help='Name of one or more collections to include, specify none to include all collections',
+ default=[],
+ action='append')
+
+ parser.add_argument('--force_app',
+ help='Necessary for using 2.0 as a source at times due to API issues. Forces the specified app(s) to be processed, even if they are not returned from the list of apps in the API call',
+ default=[],
+ action='append')
+
+ parser.add_argument('--use_name_for_collection',
+ help='Name of one or more collections to use [name] instead of [uuid] for creating entities and edges',
+ default=[],
+ action='append')
+
+ parser.add_argument('-m', '--migrate',
+ help='Specifies what to migrate: data, connections, credentials, audit or none (just iterate '
+ 'the apps/collections)',
+ type=str,
+ choices=[
+ 'data',
+ 'prune',
+ 'none',
+ 'reput',
+ 'credentials',
+ 'graph',
+ 'permissions'
+ ],
+ default='data')
+
+ parser.add_argument('-s', '--source_config',
+ help='The path to the source endpoint/org configuration file',
+ type=str,
+ default='source.json')
+
+ parser.add_argument('-d', '--target_config',
+ help='The path to the target endpoint/org configuration file',
+ type=str,
+ default='destination.json')
+
+ parser.add_argument('--redis_socket',
+ help='The path to the socket for redis to use',
+ type=str)
+
+ parser.add_argument('--limit',
+ help='The number of entities to return per query request',
+ type=int,
+ default=100)
+
+ parser.add_argument('-w', '--entity_workers',
+ help='The number of worker processes to do the migration',
+ type=int,
+ default=16)
+
+ parser.add_argument('--visit_cache_ttl',
+ help='The TTL of the cache of visiting nodes in the graph for connections',
+ type=int,
+ default=3600 * 2)
+
+ parser.add_argument('--error_retry_sleep',
+ help='The number of seconds to wait between retrieving after an error',
+ type=float,
+ default=30)
+
+ parser.add_argument('--page_sleep_time',
+ help='The number of seconds to wait between retrieving pages from the UsergridQueryIterator',
+ type=float,
+ default=0)
+
+ parser.add_argument('--entity_sleep_time',
+ help='The number of seconds to wait between retrieving pages from the UsergridQueryIterator',
+ type=float,
+ default=0)
+
+ parser.add_argument('--collection_workers',
+ help='The number of worker processes to do the migration',
+ type=int,
+ default=2)
+
+ parser.add_argument('--queue_size_max',
+ help='The max size of entities to allow in the queue',
+ type=int,
+ default=100000)
+
+ parser.add_argument('--graph_depth',
+ help='The graph depth to traverse to copy',
+ type=int,
+ default=3)
+
+ parser.add_argument('--queue_watermark_high',
+ help='The point at which publishing to the queue will PAUSE until it is at or below low watermark',
+ type=int,
+ default=25000)
+
+ parser.add_argument('--min_modified',
+ help='Break when encountering a modified date before this, per collection',
+ type=int,
+ default=0)
+
+ parser.add_argument('--max_modified',
+ help='Break when encountering a modified date after this, per collection',
+ type=long,
+ default=3793805526000)
+
+ parser.add_argument('--queue_watermark_low',
+ help='The point at which publishing to the queue will RESUME after it has reached the high watermark',
+ type=int,
+ default=5000)
+
+ parser.add_argument('--ql',
+ help='The QL to use in the filter for reading data from collections',
+ type=str,
+ default='select * order by created asc')
+ # default='select * order by created asc')
+
+ parser.add_argument('--repair_data',
+ help='Repair data when iterating/migrating graph but skipping data',
+ action='store_true')
+
+ parser.add_argument('--prune',
+ help='Prune the graph while processing (instead of the prune operation)',
+ action='store_true')
+
+ parser.add_argument('--skip_data',
+ help='Skip migrating data (useful for connections only)',
+ action='store_true')
+
+ parser.add_argument('--skip_credentials',
+ help='Skip migrating credentials',
+ action='store_true')
+
+ parser.add_argument('--skip_cache_read',
+ help='Skip reading the cache (modified timestamps and graph edges)',
+ dest='skip_cache_read',
+ action='store_true')
+
+ parser.add_argument('--skip_cache_write',
+ help='Skip updating the cache with modified timestamps of entities and graph edges',
+ dest='skip_cache_write',
+ action='store_true')
+
+ parser.add_argument('--create_apps',
+ help='Create apps at the target if they do not exist',
+ dest='create_apps',
+ action='store_true')
+
+ parser.add_argument('--nohup',
+ help='specifies not to use stdout for logging',
+ action='store_true')
+
+ parser.add_argument('--graph',
+ help='Use GRAPH instead of Query',
+ dest='graph',
+ action='store_true')
+
+ parser.add_argument('--su_username',
+ help='Superuser username',
+ required=False,
+ type=str)
+
+ parser.add_argument('--su_password',
+ help='Superuser Password',
+ required=False,
+ type=str)
+
+ parser.add_argument('--inbound_connections',
+ help='Name of the org to migrate',
+ action='store_true')
+
+ parser.add_argument('--map_app',
+ help="Multiple allowed: A colon-separated string such as 'apples:oranges' which indicates to"
+ " put data from the app named 'apples' from the source endpoint into app named 'oranges' "
+ "in the target endpoint",
+ default=[],
+ action='append')
+
+ parser.add_argument('--map_collection',
+ help="One or more colon-separated string such as 'cats:dogs' which indicates to put data from "
+ "collections named 'cats' from the source endpoint into a collection named 'dogs' in the "
+ "target endpoint, applicable globally to all apps",
+ default=[],
+ action='append')
+
+ parser.add_argument('--map_org',
+ help="One or more colon-separated strings such as 'red:blue' which indicates to put data from "
+ "org named 'red' from the source endpoint into a collection named 'blue' in the target "
+ "endpoint",
+ default=[],
+ action='append')
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+def init():
+ global config
+
+ if config.get('migrate') == 'credentials':
+
+ if config.get('su_password') is None or config.get('su_username') is None:
+ message = 'ABORT: In order to migrate credentials, Superuser parameters (su_password, su_username) are required'
+ print message
+ logger.critical(message)
+ exit()
+
+ config['collection_mapping'] = {}
+ config['app_mapping'] = {}
+ config['org_mapping'] = {}
+
+ for mapping in config.get('map_collection', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['collection_mapping'][parts[0]] = parts[1]
+ else:
+ logger.warning('Skipping Collection mapping: [%s]' % mapping)
+
+ for mapping in config.get('map_app', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['app_mapping'][parts[0]] = parts[1]
+ else:
+ logger.warning('Skipping App mapping: [%s]' % mapping)
+
+ for mapping in config.get('map_org', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['org_mapping'][parts[0]] = parts[1]
+ logger.info('Mapping Org [%s] to [%s] from mapping [%s]' % (parts[0], parts[1], mapping))
+ else:
+ logger.warning('Skipping Org mapping: [%s]' % mapping)
+
+ with open(config.get('source_config'), 'r') as f:
+ config['source_config'] = json.load(f)
+
+ with open(config.get('target_config'), 'r') as f:
+ config['target_config'] = json.load(f)
+
+ if config['exclude_collection'] is None:
+ config['exclude_collection'] = []
+
+ config['source_endpoint'] = config['source_config'].get('endpoint').copy()
+ config['source_endpoint'].update(config['source_config']['credentials'][config['org']])
+
+ target_org = config.get('org_mapping', {}).get(config.get('org'), config.get('org'))
+
+ config['target_endpoint'] = config['target_config'].get('endpoint').copy()
+ config['target_endpoint'].update(config['target_config']['credentials'][target_org])
+
+
+def wait_for(threads, label, sleep_time=60):
+ wait = True
+
+ logger.info('Starting to wait for [%s] threads with sleep time=[%s]' % (len(threads), sleep_time))
+
+ while wait:
+ wait = False
+ alive_count = 0
+
+ for t in threads:
+
+ if t.is_alive():
+ alive_count += 1
+ logger.info('Thread [%s] is still alive' % t.name)
+
+ if alive_count > 0:
+ wait = True
+ logger.info('Continuing to wait for [%s] threads with sleep time=[%s]' % (alive_count, sleep_time))
+ time.sleep(sleep_time)
+
+ logger.warn('All workers [%s] done!' % label)
+
+
+def count_bytes(entity):
+ entity_copy = entity.copy()
+
+ if 'metadata' in entity_copy:
+ del entity_copy['metadata']
+
+ entity_str = json.dumps(entity_copy)
+
+ return len(entity_str)
+
+
+def migrate_user_credentials(app, collection_name, source_entity, attempts=0):
+ # this only applies to users
+ if collection_name not in ['users', 'user'] \
+ or config.get('skip_credentials', False):
+ return False
+
+ source_identifier = get_source_identifier(source_entity)
+
+ target_app, target_collection, target_org = get_target_mapping(app, collection_name)
+
+ # get the URLs for the source and target users
+
+ source_url = user_credentials_url_template.format(org=config.get('org'),
+ app=app,
+ uuid=source_identifier,
+ **config.get('source_endpoint'))
+
+ target_url = user_credentials_url_template.format(org=target_org,
+ app=target_app,
+ uuid=source_identifier,
+ **config.get('target_endpoint'))
+
+ # this endpoint for some reason uses basic auth...
+ r = requests.get(source_url, auth=HTTPBasicAuth(config.get('su_username'), config.get('su_password')))
+
+ if r.status_code != 200:
+ logger.error('Unable to migrate credentials due to HTTP [%s] on GET URL [%s]: %s' % (
+ r.status_code, source_url, r.text))
+
+ return False
+
+ source_credentials = r.json()
+
+ logger.info('Putting credentials to [%s]...' % target_url)
+
+ r = requests.put(target_url,
+ data=json.dumps(source_credentials),
+ auth=HTTPBasicAuth(config.get('su_username'), config.get('su_password')))
+
+ if r.status_code != 200:
+ logger.error(
+ 'Unable to migrate credentials due to HTTP [%s] on PUT URL [%s]: %s' % (
+ r.status_code, target_url, r.text))
+ return False
+
+ logger.info('migrate_user_credentials | success=[%s] | app/collection/name = %s/%s/%s' % (
+ True, app, collection_name, source_entity.get('uuid')))
+
+ return True
+
+
+def check_response_status(r, url, exit_on_error=True):
+ if r.status_code != 200:
+ logger.critical('HTTP [%s] on URL=[%s]' % (r.status_code, url))
+ logger.critical('Response: %s' % r.text)
+
+ if exit_on_error:
+ exit()
+
+
+def do_operation(apps_and_collections, operation):
+ status_map = {}
+
+ logger.info('Creating queues...')
+
+ # Mac, for example, does not support the max_size for a queue in Python
+ if _platform == "linux" or _platform == "linux2":
+ entity_queue = Queue(maxsize=config.get('queue_size_max'))
+ collection_queue = Queue(maxsize=config.get('queue_size_max'))
+ collection_response_queue = Queue(maxsize=config.get('queue_size_max'))
+ else:
+ entity_queue = Queue()
+ collection_queue = Queue()
+ collection_response_queue = Queue()
+
+ logger.info('Starting entity_workers...')
+
+ collection_count = 0
+ # create the entity workers, but only start them (later) if there is work to do
+ entity_workers = [EntityWorker(entity_queue, operation) for x in xrange(config.get('entity_workers'))]
+
+ # create the collection workers, but only start them (later) if there is work to do
+ collection_workers = [CollectionWorker(collection_queue, entity_queue, collection_response_queue) for x in
+ xrange(config.get('collection_workers'))]
+
+ status_listener = StatusListener(collection_response_queue, entity_queue)
+
+ try:
+ # for each app, publish the (app_name, collection_name) to the queue.
+ # this is received by a collection worker who iterates the collection and publishes
+ # entities into a queue. These are received by an individual entity worker which
+ # executes the specified operation on the entity
+
+ for app, app_data in apps_and_collections.get('apps', {}).iteritems():
+ logger.info('Processing app=[%s]' % app)
+
+ status_map[app] = {
+ 'iteration_started': str(datetime.datetime.now()),
+ 'max_created': -1,
+ 'max_modified': -1,
+ 'min_created': 1584946416000,
+ 'min_modified': 1584946416000,
+ 'count': 0,
+ 'bytes': 0,
+ 'collections': {}
+ }
+
+ # iterate the collections which are returned.
+ for collection_name in app_data.get('collections'):
+ logger.info('Publishing app / collection: %s / %s' % (app, collection_name))
+
+ collection_count += 1
+ collection_queue.put((app, collection_name))
+
+ logger.info('Finished publishing [%s] collections for app [%s] !' % (collection_count, app))
+
+ # only start the threads if there is work to do
+ if collection_count > 0:
+ status_listener.start()
+
+ # start the worker processes which will iterate the collections
+ [w.start() for w in collection_workers]
+
+ # start the worker processes which will do the work of migrating
+ [w.start() for w in entity_workers]
+
+ # allow collection workers to finish
+ wait_for(collection_workers, label='collection_workers', sleep_time=60)
+
+ # allow entity workers to finish
+ wait_for(entity_workers, label='entity_workers', sleep_time=60)
+
+ status_listener.terminate()
+
+ except KeyboardInterrupt:
+ logger.warning('Keyboard Interrupt, aborting...')
+ entity_queue.close()
+ collection_queue.close()
+ collection_response_queue.close()
+
+ [os.kill(super(EntityWorker, p).pid, signal.SIGINT) for p in entity_workers]
+ [os.kill(super(CollectionWorker, p).pid, signal.SIGINT) for p in collection_workers]
+ os.kill(super(StatusListener, status_listener).pid, signal.SIGINT)
+
+ [w.terminate() for w in entity_workers]
+ [w.terminate() for w in collection_workers]
+ status_listener.terminate()
+
+ logger.info('entity_workers DONE!')
+
+
+def filter_apps_and_collections(org_apps):
+ app_collecitons = {
+ 'apps': {
+
+ }
+ }
+
+ try:
+ selected_apps = config.get('app')
+
+ # iterate the apps retrieved from the org
+ for org_app in sorted(org_apps.keys()):
+ logger.info('Found SOURCE App: %s' % org_app)
+
+ time.sleep(3)
+
+ for org_app in sorted(org_apps.keys()):
+ parts = org_app.split('/')
+ app = parts[1]
+
+ # if apps are specified and the current app is not in the list, skip it
+ if selected_apps and len(selected_apps) > 0 and app not in selected_apps:
+ logger.warning('Skipping app [%s] not included in process list [%s]' % (app, selected_apps))
+ continue
+
+ app_collecitons['apps'][app] = {
+ 'collections': []
+ }
+
+ # get the list of collections from the source org/app
+ source_app_url = app_url_template.format(org=config.get('org'),
+ app=app,
+ **config.get('source_endpoint'))
+ logger.info('GET %s' % source_app_url)
+
+ r_collections = session_source.get(source_app_url)
+
+ collection_attempts = 0
+
+ # sometimes this call was not working so I put it in a loop to force it...
+ while r_collections.status_code != 200 and collection_attempts < 5:
+ collection_attempts += 1
+ logger.warning('FAILED: GET (%s) [%s] URL: %s' % (r_collections.elapsed, r_collections.status_code,
+ source_app_url))
+ time.sleep(DEFAULT_RETRY_SLEEP)
+ r_collections = session_source.get(source_app_url)
+
+ if collection_attempts >= 5:
+ logger.critical('Unable to get collections at URL %s, skipping app' % source_app_url)
+ continue
+
+ app_response = r_collections.json()
+
+ logger.info('App Response: ' + json.dumps(app_response))
+
+ app_entities = app_response.get('entities', [])
+
+ if len(app_entities) > 0:
+ app_entity = app_entities[0]
+ collections = app_entity.get('metadata', {}).get('collections', {})
+ logger.info('App=[%s] starting Collections=[%s]' % (app, collections))
+
+ app_collecitons['apps'][app]['collections'] = [c for c in collections if include_collection(c)]
+ logger.info('App=[%s] filtered Collections=[%s]' % (app, collections))
+
+ except:
+ print traceback.format_exc()
+
+ return app_collecitons
+
+
+def confirm_target_org_apps(apps_and_collections):
+ for app in apps_and_collections.get('apps'):
+
+ # it is possible to map source orgs and apps to differently named targets. This gets the
+ # target names for each
+ target_org = config.get('org_mapping', {}).get(config.get('org'), config.get('org'))
+ target_app = config.get('app_mapping', {}).get(app, app)
+
+ # Check that the target Org/App exists. If not, move on to the next
+ target_app_url = app_url_template.format(org=target_org,
+ app=target_app,
+ **config.get('target_endpoint'))
+ logger.info('GET %s' % target_app_url)
+ r_target_apps = session_target.get(target_app_url)
+
+ if r_target_apps.status_code != 200:
+
+ if config.get('create_apps', DEFAULT_CREATE_APPS):
+ create_app_url = org_management_app_url_template.format(org=target_org,
+ app=target_app,
+ **config.get('target_endpoint'))
+ app_request = {'name': target_app}
+ r = session_target.post(create_app_url, data=json.dumps(app_request))
+
+ if r.status_code != 200:
+ logger.critical('--create_apps specified and unable to create app [%s] at URL=[%s]: %s' % (
+ target_app, create_app_url, r.text))
+ logger.critical('Process will now exit')
+ exit()
+ else:
+ logger.warning('Created app=[%s] at URL=[%s]: %s' % (target_app, create_app_url, r.text))
+ else:
+ logger.critical('Target application DOES NOT EXIST at [%s] URL=%s' % (
+ r_target_apps.status_code, target_app_url))
+ continue
+
+
+def main():
+ global config, cache
+
+ config = parse_args()
+ init()
+ init_logging()
+
+ logger.warn('Script starting')
+
+ try:
+ if config.get('redis_socket') is not None:
+ cache = redis.Redis(unix_socket_path=config.get('redis_socket'))
+
+ else:
+ # this does not try to connect to redis
+ cache = redis.StrictRedis(host='localhost', port=6379, db=0)
+
+ # this is necessary to test the connection to redis
+ cache.get('usergrid')
+
+ except:
+ logger.error(
+ 'Error connecting to Redis cache, consider using Redis to be able to optimize the migration process...')
+ logger.error(
+ 'Error connecting to Redis cache, consider using Redis to be able to optimize the migration process...')
+
+ time.sleep(3)
+
+ config['use_cache'] = False
+ config['skip_cache_read'] = True
+ config['skip_cache_write'] = True
+
+ org_apps = {
+ }
+
+ force_apps = config.get('force_app', [])
+
+ if force_apps is not None and len(force_apps) > 0:
+ logger.warn('Forcing only the following apps to be processed: %s' % force_apps)
+
+ for app in force_apps:
+ ke
<TRUNCATED>
[47/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/545/head' of github.com:apache/usergrid
Posted by mr...@apache.org.
Merge commit 'refs/pull/545/head' of github.com:apache/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2ba664cb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2ba664cb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2ba664cb
Branch: refs/heads/master
Commit: 2ba664cb5888460858ad973ab858b4eea1523e68
Parents: 1e11464 5a0756a
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Aug 1 09:51:57 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Aug 1 09:51:57 2016 -0700
----------------------------------------------------------------------
.../Usergrid.Sdk.IntegrationTests/BaseTest.cs | 23 +++++++++++++++++---
.../EntityPagingTests.cs | 2 +-
.../Usergrid.Sdk.IntegrationTests/GroupTests.cs | 4 ++--
.../Usergrid.Sdk.IntegrationTests/LoginTests.cs | 12 +++++-----
.../Usergrid.Sdk.IntegrationTests.dll.config | 1 +
5 files changed, 30 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[32/50] [abbrv] usergrid git commit: Merge branch 'master' into
apigee-sso-provider
Posted by mr...@apache.org.
Merge branch 'master' into apigee-sso-provider
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9d78b59b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9d78b59b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9d78b59b
Branch: refs/heads/master
Commit: 9d78b59bf613d1c2951a089e00495859954228b7
Parents: d6941b7 acd8719
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Jul 26 14:58:25 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Jul 26 14:58:25 2016 -0700
----------------------------------------------------------------------
.../java/client/model/UsergridEntity.java | 2 +-
stack/query-validator/pom.xml | 4 +-
.../query/validator/ApiServerRunner.java | 89 +++++++-------------
3 files changed, 33 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
[29/50] [abbrv] usergrid git commit: Code review changes
Posted by mr...@apache.org.
Code review changes
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f8d5713b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f8d5713b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f8d5713b
Branch: refs/heads/master
Commit: f8d5713b620a5f2df6379f4e9073fa9edbebdafa
Parents: 6d88bcf
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Tue Jul 26 14:09:06 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Tue Jul 26 14:09:06 2016 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +++
.../rest/management/ManagementResource.java | 10 +++-------
.../organizations/OrganizationsResource.java | 18 ++++++++++--------
.../cassandra/ManagementServiceImpl.java | 11 +++++------
4 files changed, 21 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f8d5713b/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 371b251..23da7e9 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -682,6 +682,9 @@ usergrid.test-account.admin-user.password=test
#
#
+# Set a flag to allow public org registrations
+usergrid.management.allow-public-registrations=true
+
# Set the requirements for activiation and confirmations
usergrid.sysadmin.approve.users=false
usergrid.sysadmin.approve.organizations=false
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f8d5713b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index c5d7c77..285a22e 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -299,13 +299,9 @@ public class ManagementResource extends AbstractContextResource {
if ( user == null ) {
-
- //if ( !me ) { // if not lightweight-auth, i.e. /management/me then...
-
- // make sure authentication is allowed considering
- // external token validation configuration (UG Central SSO)
- ensureAuthenticationAllowed( username, grant_type );
- // }
+ // make sure authentication is allowed considering
+ // external token validation configuration (UG Central SSO)
+ ensureAuthenticationAllowed( username, grant_type );
if ( authorization != null ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f8d5713b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
index 823ebcc..b75ca60 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationsResource.java
@@ -40,8 +40,6 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
import java.util.*;
-import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER_URL;
-
@Component( "org.apache.usergrid.rest.management.organizations.OrganizationsResource" )
@Scope( "prototype" )
@@ -187,12 +185,16 @@ public class OrganizationsResource extends AbstractContextResource {
String email, String password, Map<String, Object> userProperties,
Map<String, Object> orgProperties, String callback ) throws Exception {
- if ( tokens.isExternalSSOProviderEnabled() ) {
- //let superuser add an org even if external SSO Provider is enabled.
- if(!userServiceAdmin(null) ) { // what should the username be ?
- throw new IllegalArgumentException("Organization / Admin Users must be created via " +
- properties.getProperty(USERGRID_EXTERNAL_PROVIDER_URL));
- }
+ // Providing no password in this request signifies that an existing admin users should be associated to the
+ // newly requested organization.
+
+ // Always let the sysadmin create an org, but otherwise follow the behavior specified with
+ // the property 'usergrid.management.allow-public-registration'
+ if ( ( System.getProperty("usergrid.management.allow-public-registration") != null
+ && !Boolean.valueOf(System.getProperty("usergrid.management.allow-public-registration"))
+ && !userServiceAdmin(null) ) ) {
+
+ throw new IllegalArgumentException("Public organization registration is disabled");
}
Preconditions
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f8d5713b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index e8bf0ec..97febdf 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -549,20 +549,19 @@ public class ManagementServiceImpl implements ManagementService {
return null;
}
- // irrespective of it being sso enabled or not , if its a super user request it will try to fetch user if no password is passed.
- if(password == null && SubjectUtils.isServiceAdmin()){
+ // sysadmin can omit password field in the request and that will try to fetch an existing admin user to
+ // associate to the requested organization
+ if((password == null || password.isEmpty()) && SubjectUtils.isServiceAdmin()){
user = getAdminUserByEmail(email);
if(user == null ){
throw new IllegalArgumentException("Password should be sent in the request or should be a valid admin user email.");
}
}
- else if(password == null ){ //for existing workflow.
- throw new IllegalArgumentException("Password should be sent in the request.");
- }
if(user == null) {
- if ((tokens.isExternalSSOProviderEnabled() && SubjectUtils.isServiceAdmin()) || areActivationChecksDisabled()) {
+ // if external SSO is enabled and we're adding a user to an org, auto activate the user
+ if (tokens.isExternalSSOProviderEnabled() || areActivationChecksDisabled()) {
user = createAdminUserInternal(null, username, name, email, password, true, false, userProperties);
} else {
user = createAdminUserInternal(null, username, name, email, password, activated, disabled, userProperties);
[10/50] [abbrv] usergrid git commit: Merge branch
'apigee-sso-provider' of https://github.com/apache/usergrid into
SSO2-Usergrid
Posted by mr...@apache.org.
Merge branch 'apigee-sso-provider' of https://github.com/apache/usergrid into SSO2-Usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3ba4e596
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3ba4e596
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3ba4e596
Branch: refs/heads/master
Commit: 3ba4e596c5bccffea4d7fd481e55c1590886322e
Parents: 2c59bd8 8413f21
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Thu Jul 7 17:56:39 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Thu Jul 7 17:56:39 2016 -0700
----------------------------------------------------------------------
.../usergrid/rest/management/users/UserResource.java | 3 ++-
.../security/shiro/filters/BasicAuthSecurityFilter.java | 3 +++
.../shiro/filters/ClientCredentialsSecurityFilter.java | 4 ++++
.../shiro/filters/OAuth2AccessTokenSecurityFilter.java | 4 ++++
.../rest/security/shiro/filters/SecurityFilter.java | 12 ++++++++++++
5 files changed, 25 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3ba4e596/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index 89725b8,36dbdde..b16e85c
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@@ -199,10 -195,9 +199,11 @@@ public class UserResource extends Abstr
ApiResponse response = createApiResponse();
response.setAction( "get admin user" );
- //String token = management.getAccessTokenForAdminUser( user.getUuid(), ttl );
+// commenting out creation of token each time and setting the token value to the one sent in the request.
+// String token = management.getAccessTokenForAdminUser( user.getUuid(), ttl );
++
Map<String, Object> userOrganizationData = management.getAdminUserOrganizationData( user, !shallow );
- userOrganizationData.put( "token", token );
+ //userOrganizationData.put( "token", token );
response.setData( userOrganizationData );
response.setSuccess();
[09/50] [abbrv] usergrid git commit: adding a check to ensure
activationConfirmation emails are not sent for external sos provider enabled
requests.
Posted by mr...@apache.org.
adding a check to ensure activationConfirmation emails are not sent for external sos provider enabled requests.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2c59bd8e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2c59bd8e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2c59bd8e
Branch: refs/heads/master
Commit: 2c59bd8ee5c4313142a929e2f8301b83c00f3a1b
Parents: 66ca27c
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Thu Jul 7 17:52:46 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Thu Jul 7 17:52:46 2016 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/rest/management/users/UserResource.java | 3 ++-
.../usergrid/management/cassandra/ManagementServiceImpl.java | 5 ++++-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2c59bd8e/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index 1c5bcdf..89725b8 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -199,7 +199,8 @@ public class UserResource extends AbstractContextResource {
ApiResponse response = createApiResponse();
response.setAction( "get admin user" );
- String token = management.getAccessTokenForAdminUser( user.getUuid(), ttl );
+// commenting out creation of token each time and setting the token value to the one sent in the request.
+// String token = management.getAccessTokenForAdminUser( user.getUuid(), ttl );
Map<String, Object> userOrganizationData = management.getAdminUserOrganizationData( user, !shallow );
userOrganizationData.put( "token", token );
response.setData( userOrganizationData );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2c59bd8e/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index b062f84..9637dff 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -904,10 +904,13 @@ public class ManagementServiceImpl implements ManagementService {
user.getEmail(), user.getConfirmed(), user.getActivated(), user.getDisabled(),
user.getDynamicProperties(), true );
+
// special case for sysadmin and test account only
if ( !user.getEmail().equals( properties.getProperty( PROPERTIES_SYSADMIN_LOGIN_EMAIL ) )
&& !user.getEmail().equals( properties .getProperty( PROPERTIES_TEST_ACCOUNT_ADMIN_USER_EMAIL ) ) ) {
- this.startAdminUserActivationFlow( organizationId, userInfo );
+ if(!tokens.isExternalSSOProviderEnabled()) {
+ this.startAdminUserActivationFlow(organizationId, userInfo);
+ }
}
return userInfo;
[08/50] [abbrv] usergrid git commit: Allow CORS pre-flight requests
to come through unauthenticated ( bad creds shouldn't stop browsers from
trying the real request ).
Posted by mr...@apache.org.
Allow CORS pre-flight requests to come through unauthenticated ( bad creds shouldn't stop browsers from trying the real request ).
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8413f212
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8413f212
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8413f212
Branch: refs/heads/master
Commit: 8413f212ee7bdfdd729d4f3f7d93200362e01751
Parents: 8d79d36
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jul 7 17:47:52 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jul 7 17:47:52 2016 -0700
----------------------------------------------------------------------
.../security/shiro/filters/BasicAuthSecurityFilter.java | 3 +++
.../shiro/filters/ClientCredentialsSecurityFilter.java | 4 ++++
.../shiro/filters/OAuth2AccessTokenSecurityFilter.java | 4 ++++
.../rest/security/shiro/filters/SecurityFilter.java | 12 ++++++++++++
4 files changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8413f212/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/BasicAuthSecurityFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/BasicAuthSecurityFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/BasicAuthSecurityFilter.java
index a5d7272..5594a1c 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/BasicAuthSecurityFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/BasicAuthSecurityFilter.java
@@ -49,6 +49,9 @@ public class BasicAuthSecurityFilter extends SecurityFilter {
logger.trace("Filtering: {}", request.getUriInfo().getBaseUri());
}
+ if( bypassSecurityCheck(request) ){
+ return;
+ }
Map<String, String> auth_types = getAuthTypes( request );
if ( ( auth_types == null ) || !auth_types.containsKey( AUTH_BASIC_TYPE ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8413f212/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/ClientCredentialsSecurityFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/ClientCredentialsSecurityFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/ClientCredentialsSecurityFilter.java
index 83e53c1..486d105 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/ClientCredentialsSecurityFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/ClientCredentialsSecurityFilter.java
@@ -55,6 +55,10 @@ public class ClientCredentialsSecurityFilter extends SecurityFilter {
logger.trace("Filtering: {}", request.getUriInfo().getBaseUri());
}
+ if( bypassSecurityCheck(request) ){
+ return;
+ }
+
String clientId = httpServletRequest.getParameter( "client_id" );
String clientSecret = httpServletRequest.getParameter( "client_secret" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8413f212/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
index 03da0e8..ca040e8 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/OAuth2AccessTokenSecurityFilter.java
@@ -74,6 +74,10 @@ public class OAuth2AccessTokenSecurityFilter extends SecurityFilter implements C
logger.trace("Filtering: {}", request.getUriInfo().getBaseUri());
}
+ if( bypassSecurityCheck(request) ){
+ return;
+ }
+
try {
try {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8413f212/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java
index e0dadba..1c06aed 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/security/shiro/filters/SecurityFilter.java
@@ -132,4 +132,16 @@ public abstract class SecurityFilter implements ContainerRequestFilter {
}
return auth_types;
}
+
+ public static boolean bypassSecurityCheck( ContainerRequestContext request ){
+
+ // if this is a CORS Pre-Flight request, we can skip the security check
+ // OPTIONS requests do not have access into Usergrid data, Jersey default handles these requests
+ if( request.getMethod().equalsIgnoreCase("options")){
+ return true;
+ }
+
+ return false;
+
+ }
}
[39/50] [abbrv] usergrid git commit: Merge branch 'master' into
apigee-sso-provider
Posted by mr...@apache.org.
Merge branch 'master' into apigee-sso-provider
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ddde8ddd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ddde8ddd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ddde8ddd
Branch: refs/heads/master
Commit: ddde8dddb60e3e77e7c9ad40201c3727fc77eafc
Parents: 9d78b59 4558dc9
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 27 13:53:44 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 27 13:53:44 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/query/validator/AbstractQueryIT.java | 3 ++-
.../org/apache/usergrid/query/validator/users/UserQueryIT.java | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[22/50] [abbrv] usergrid git commit: Cleanup. Add tests.
Posted by mr...@apache.org.
Cleanup. Add tests.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6d88bcfd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6d88bcfd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6d88bcfd
Branch: refs/heads/master
Commit: 6d88bcfdccc458b382b16483b5d3048df699262c
Parents: 5fcad44
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Tue Jul 26 11:49:42 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Tue Jul 26 11:49:42 2016 -0700
----------------------------------------------------------------------
stack/rest/pom.xml | 8 +
.../rest/management/ManagementResource.java | 13 +-
.../rest/management/ExternalSSOEnabledIT.java | 175 +++++++++++++++++++
.../rest/management/ManagementResourceIT.java | 24 ++-
.../security/sso/ApigeeSSO2Provider.java | 28 +--
.../tokens/cassandra/TokenServiceImpl.java | 3 +
6 files changed, 233 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6d88bcfd/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index 89d14b4..580814c 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -306,6 +306,14 @@
<version>${jackson-2-version}</version>
</dependency>
+ <!-- for mocking the Apigee SSO Service -->
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<!-- databinding; ObjectMapper, JsonNode and related classes are here -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6d88bcfd/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 7217c49..c5d7c77 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -385,9 +385,14 @@ public class ManagementResource extends AbstractContextResource {
//moved the check for sso enabled form MangementServiceImpl since was unable to get the current user there to check if its super user.
if( tokens.isExternalSSOProviderEnabled()
- && !userServiceAdmin(user.getUsername()) ){
- throw new RuntimeException("SSO Integration is enabled, Admin users must login via provider: "+
- properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER));
+ && !userServiceAdmin(username) ){
+ OAuthResponse response =
+ OAuthResponse.errorResponse( SC_BAD_REQUEST ).setError( OAuthError.TokenResponse.INVALID_GRANT )
+ .setErrorDescription( "SSO Integration is enabled, Admin users must login via provider: "+
+ properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER) ).buildJSONMessage();
+ return Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
+ .entity( wrapWithCallback( response.getBody(), callback ) ).build();
+
}
String token = management.getAccessTokenForAdminUser( user.getUuid(), ttl );
@@ -616,7 +621,7 @@ public class ManagementResource extends AbstractContextResource {
if ( tokens.isExternalSSOProviderEnabled() ) {
// when external tokens enabled then only superuser can obtain an access token
- if ( userServiceAdmin(username)) {
+ if ( !userServiceAdmin(username)) {
// this guy is not the superuser
throw new IllegalArgumentException( "Admin Users must login via " +
properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6d88bcfd/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExternalSSOEnabledIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExternalSSOEnabledIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExternalSSOEnabledIT.java
new file mode 100644
index 0000000..75fa0b5
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExternalSSOEnabledIT.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.rest.management;
+
+import io.jsonwebtoken.Jwts;
+import io.jsonwebtoken.SignatureAlgorithm;
+import io.jsonwebtoken.impl.crypto.RsaProvider;
+import org.apache.commons.collections4.map.HashedMap;
+import org.apache.usergrid.cassandra.SpringResource;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.rest.test.resource.AbstractRestIT;
+import org.apache.usergrid.rest.test.resource.RestClient;
+import org.apache.usergrid.rest.test.resource.model.ApiResponse;
+import org.apache.usergrid.rest.test.resource.model.Entity;
+import org.apache.usergrid.security.sso.ApigeeSSO2Provider;
+import org.codehaus.jackson.JsonNode;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.security.*;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+/**
+ * Created by ayeshadastagiri on 7/20/16.
+ */
+@Ignore("Need to figure out a way to set the public key for Mock server.")
+public class ExternalSSOEnabledIT extends AbstractRestIT {
+
+ Key key;
+ PublicKey publicKey;
+ PrivateKey privateKey;
+ String compactJws;
+ String username = "SSOadminuser" + UUIDUtils.newTimeUUID();
+ ApigeeSSO2Provider apigeeSSO2ProviderTest;
+ //SSO2 implementation
+ public static final String USERGRID_EXTERNAL_SSO_ENABLED = "usergrid.external.sso.enabled";
+ public static final String USERGRID_EXTERNAL_PROVIDER = "usergrid.external.sso.provider";
+
+ public ExternalSSOEnabledIT() throws Exception {
+
+ }
+
+ @Before
+ public void setup() throws NoSuchAlgorithmException {
+ generateKey();
+ }
+
+ private void generateKey() {
+ KeyPair kp = RsaProvider.generateKeyPair(1024);
+ publicKey = kp.getPublic();
+ privateKey = kp.getPrivate();
+ }
+
+ private String genrateToken(){
+ Map<String, Object> claims = new HashedMap<String, Object>();
+ claims.put("jti","c7df0339-3847-450b-a925-628ef237953a");
+ claims.put("sub","b6d62259-217b-4e96-8f49-e00c366e4fed");
+ claims.put("scope","size = 5");
+ claims.put("client_id", "edgecli");
+ claims.put("azp","edgecli");
+ claims.put("grant_type" ,"password");
+ claims.put("user_id","b6d62259-217b-4e96-8f49-e00c366e4fed");
+ claims.put( "origin","usergrid");
+ claims.put("user_name","AyeshaSSOUser");
+ claims.put("email", "adastagiri+ssotesting@apigee.com");
+ claims.put( "rev_sig","dfe5d0d3");
+ claims.put("iat","1466550862");
+ claims.put("exp", System.currentTimeMillis() + 1000);
+ claims.put("iss", "https://login.apigee.com/oauth/token");
+ claims.put( "zid","uaa");
+ claims.put( "aud"," size = 6");
+ claims.put("grant_type","password");
+
+ String jwt = Jwts.builder().setClaims(claims).signWith(SignatureAlgorithm.RS256, privateKey).compact();
+ return jwt;
+
+ }
+
+ @Test
+ public void SuperUserTestsFor() throws NoSuchAlgorithmException {
+
+ // create a admin user.
+ RestClient restClient = clientSetup.getRestClient();
+
+ //Create adminUser values
+ Entity adminUserPayload = new Entity();
+ adminUserPayload.put("username", "TestUser");
+ adminUserPayload.put("name", username);
+ adminUserPayload.put("email", "adastagiri+ssotesting@apigee.com");
+ adminUserPayload.put("password", username);
+
+ //create adminUser
+ ApiResponse adminUserEntityResponse = management().orgs().org(clientSetup.getOrganizationName()).users().post(ApiResponse.class, adminUserPayload);
+
+ Entity adminUserResponse = new Entity(adminUserEntityResponse);
+ //verify that the response contains the correct data
+ assertNotNull(adminUserResponse);
+ assertEquals("TestUser", adminUserResponse.get("username"));
+
+ Map<String, String> props = new HashMap<String, String>();
+
+ props.put( USERGRID_EXTERNAL_SSO_ENABLED, "true" );
+ props.put( USERGRID_EXTERNAL_PROVIDER, "apigee" );
+ pathResource( "testproperties" ).post( props );
+
+ // /management/me --> superuser and query params --> Generate a super usesr token.
+ Map<String, Object> loginInfo = new HashMap<String, Object>() {{
+ put( "username", "superuser" );
+ put( "password", "superpassword" );
+ put( "grant_type", "password" );
+ }};
+ ApiResponse postResponse2 = pathResource( "management/token" ).post( false,ApiResponse.class,loginInfo );
+ assertTrue(postResponse2.getAccessToken() != null );
+
+
+ // /orgs create an org with superuser credentials.
+ // /management/me --> superuser and query params --> Generate a super usesr token.
+ Map<String, Object> orgDetails = new HashMap<String, Object>() {{
+ put( "email", "adastagiri+ssotesting@apigee.com" );
+ put( "name", "testuser" );
+ put( "organization", username );
+ }};
+
+ context().getToken().put("access_token",postResponse2.getAccessToken());
+ postResponse2 = pathResource( "management/orgs" ).post( true,ApiResponse.class,orgDetails);
+ assertTrue(postResponse2.getData() != null);
+
+ postResponse2 = pathResource("management/orgs").get(ApiResponse.class,true);
+ assertTrue(postResponse2 != null);
+
+
+ compactJws = genrateToken();
+
+ SpringResource.getInstance().getAppContext().getBean(ApigeeSSO2Provider.class).setPublicKey( publicKey );
+ context().getToken().put("access_token",compactJws);
+ // /management/me --> admin user and jwt token. Return the user information and "token" should have jwt token.
+ JsonNode responseToken = management().me().get(JsonNode.class,true);
+ assertTrue(responseToken.get("access_token") != null);
+
+
+ // /management/me --> admin and query params --> Generate a super usesr token.
+ Map<String, Object> loginInfo1 = new HashMap<String, Object>() {{
+ put( "username", "TestUser" );
+ put( "password", username );
+ put( "grant_type", "password" );
+ }};
+
+ // /managment/token -> adminusername and password --> should fail.
+ ApiResponse postResponse1 = pathResource("management/token").post(false, ApiResponse.class,loginInfo1);
+// fail( "SSO Integration is enabled, Admin users must login via provider: "+ USERGRID_EXTERNAL_PROVIDER_URL);
+
+
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6d88bcfd/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index 1fe4f01..21c62ba 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -38,6 +38,7 @@ import java.io.IOException;
import java.util.*;
import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER_URL;
+import static org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl.USERGRID_EXTERNAL_SSO_ENABLED;
import static org.apache.usergrid.utils.MapUtils.hashMap;
import static org.junit.Assert.*;
@@ -671,6 +672,7 @@ public class ManagementResourceIT extends AbstractRestIT {
String suToken = clientSetup.getSuperuserToken().getAccessToken();
Map<String, String> props = new HashMap<String, String>();
+ props.put(USERGRID_EXTERNAL_SSO_ENABLED, "true");
props.put( USERGRID_EXTERNAL_PROVIDER_URL, getBaseURI().toURL().toExternalForm() );
pathResource( "testproperties" ).post( props );
@@ -685,7 +687,7 @@ public class ManagementResourceIT extends AbstractRestIT {
put( "grant_type", "password" );
}};
ApiResponse postResponse = pathResource( "management/token" ).post( false, ApiResponse.class, loginInfo );
- fail( "Login as Admin User must fail when validate external tokens is enabled" );
+ fail( "SSO Integration is enabled, Admin users must login via provider: "+ USERGRID_EXTERNAL_PROVIDER_URL);
} catch (ClientErrorException actual) {
assertEquals( 400, actual.getResponse().getStatus() );
@@ -709,11 +711,31 @@ public class ManagementResourceIT extends AbstractRestIT {
String accessToken = postResponse2.getAccessToken();
assertNotNull( accessToken );
+ //Superuser : GET -> get tokenInfo with access_token
+ ApiResponse getResponse3 = pathResource("management/me").get(ApiResponse.class,new QueryParameters()
+ .addParam("grant_type", "password").addParam("password", "superpassword")
+ .addParam("username", "superuser"),false);
+
+ assertNotNull(getResponse3.getAccessToken());
+
+ //Superuser : POST -> Add org using super user credentials.
+ Map<String, Object> orgAdminUserInfo = new HashMap<String, Object>() {{
+ put( "username", username+"test" );
+ put("password","RandomPassword");
+ put("email",username+"@gmail.com");
+ put( "organization", username+"RandomOrgName" );
+ }};
+ ApiResponse postResponse4 = pathResource("management/orgs")
+ .post(false,orgAdminUserInfo,new QueryParameters().addParam("access_token",getResponse3.getAccessToken()));
+ assertNotNull(postResponse4.getData());
+
+
} finally {
// turn off validate external tokens by un-setting the usergrid.central.url
props.put( USERGRID_EXTERNAL_PROVIDER_URL, "" );
+ props.put(USERGRID_EXTERNAL_SSO_ENABLED, "");
pathResource( "testproperties" ).post( props );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6d88bcfd/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
index 61a1601..d48495d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
@@ -57,7 +57,7 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
protected Properties properties;
protected ManagementService management;
protected Client client;
- protected String publicKey;
+ protected PublicKey publicKey;
public static final String USERGRID_EXTERNAL_PUBLICKEY_URL = "usergrid.external.sso.url";
@@ -67,12 +67,17 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
client = ClientBuilder.newClient(clientConfig);
}
- public String getPublicKey(String keyUrl) {
+ public PublicKey getPublicKey(String keyUrl) {
if(keyUrl != null && !keyUrl.isEmpty()) {
try {
Map<String, Object> publicKey = client.target(keyUrl).request().get(Map.class);
- return publicKey.get(RESPONSE_PUBLICKEY_VALUE).toString().split("----\n")[1].split("\n---")[0];
+ String ssoPublicKey = publicKey.get(RESPONSE_PUBLICKEY_VALUE).toString().split("----\n")[1].split("\n---")[0];
+ byte[] publicBytes = decodeBase64(ssoPublicKey);
+ X509EncodedKeySpec keySpec = new X509EncodedKeySpec(publicBytes);
+ KeyFactory keyFactory = KeyFactory.getInstance("RSA");
+ PublicKey pubKey = keyFactory.generatePublic(keySpec);
+ return pubKey;
}
catch(Exception e){
throw new IllegalArgumentException("error getting public key");
@@ -140,7 +145,7 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
return properties.getProperty(USERGRID_EXTERNAL_PUBLICKEY_URL);
}
- public Jws<Claims> getClaimsForKeyUrl(String token, String ssoPublicKey) throws NoSuchAlgorithmException, InvalidKeySpecException, BadTokenException {
+ public Jws<Claims> getClaimsForKeyUrl(String token, PublicKey ssoPublicKey) throws NoSuchAlgorithmException, InvalidKeySpecException, BadTokenException {
Jws<Claims> claims = null;
if(ssoPublicKey == null){
@@ -148,14 +153,8 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
"token in order to verify signature.");
}
-
- byte[] publicBytes = decodeBase64(ssoPublicKey);
- X509EncodedKeySpec keySpec = new X509EncodedKeySpec(publicBytes);
- KeyFactory keyFactory = KeyFactory.getInstance("RSA");
- PublicKey pubKey = keyFactory.generatePublic(keySpec);
-
try {
- claims = Jwts.parser().setSigningKey(pubKey).parseClaimsJws(token);
+ claims = Jwts.parser().setSigningKey(ssoPublicKey).parseClaimsJws(token);
} catch (SignatureException se) {
if(logger.isDebugEnabled()) {
logger.debug("Signature was invalid for Apigee JWT: {} and key: {}", token, ssoPublicKey);
@@ -196,10 +195,13 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
throw new ExpiredTokenException(String.format("Token expired %d millisecons ago.", expirationDelta ));
}
-
}
+ public void setPublicKey( PublicKey publicKeyArg){
+ this.publicKey = publicKeyArg;
+ }
+
@Autowired
public void setManagement(ManagementService management) {
this.management = management;
@@ -208,6 +210,6 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
@Autowired
public void setProperties(Properties properties) {
this.properties = properties;
- this.publicKey = getPublicKey(getExternalSSOUrl());
+ this.publicKey = getPublicKey(getExternalSSOUrl());
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6d88bcfd/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
index 62d454a..46ea7fe 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
@@ -328,6 +328,9 @@ public class TokenServiceImpl implements TokenService {
try{
uuid = getUUIDForToken( token );
}
+ catch (ExpiredTokenException expiredTokenException){
+ throw new ExpiredTokenException(expiredTokenException.getMessage());
+ }
catch(Exception e){
// If the token doesn't parse as a Usergrid token, see if an external provider other than Usergrid is
[28/50] [abbrv] usergrid git commit: Initial checkin for Python
Utilities and SDK
Posted by mr...@apache.org.
Initial checkin for Python Utilities and SDK
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/32f9e55d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/32f9e55d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/32f9e55d
Branch: refs/heads/master
Commit: 32f9e55da01c92dbda81a88a465f088405259faa
Parents: 91abfd8
Author: Jeff West <jw...@apigee.com>
Authored: Tue Jul 26 13:26:36 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Tue Jul 26 13:26:36 2016 -0700
----------------------------------------------------------------------
sdks/python/.gitignore | 57 +
sdks/python/LICENSE | 202 ++
sdks/python/README.md | 16 +
sdks/python/sample_app.py | 68 +
sdks/python/setup.py | 39 +
sdks/python/usergrid/UsergridApplication.py | 62 +
sdks/python/usergrid/UsergridAuth.py | 103 +
sdks/python/usergrid/UsergridClient.py | 398 ++++
sdks/python/usergrid/UsergridCollection.py | 77 +
sdks/python/usergrid/UsergridConnection.py | 26 +
sdks/python/usergrid/UsergridError.py | 17 +
sdks/python/usergrid/UsergridOrganization.py | 31 +
sdks/python/usergrid/UsergridQueryIterator.py | 155 ++
sdks/python/usergrid/__init__.py | 37 +
sdks/python/usergrid/app_templates.py | 36 +
sdks/python/usergrid/management_templates.py | 25 +
utils/usergrid-util-python/.gitignore | 61 +
utils/usergrid-util-python/LICENSE | 22 +
utils/usergrid-util-python/README.md | 15 +
.../es_tools/alias_mover.py | 129 ++
.../es_tools/cluster_shard_allocation.py | 89 +
.../es_tools/command_sender.py | 42 +
.../es_tools/es_index_iterator_reindexer.py | 107 +
.../es_tools/es_searcher.py | 24 +
.../es_tools/index_deleter.py | 68 +
.../es_tools/index_iterator_size_checker.py | 270 +++
.../es_tools/index_prefix_checker.py | 81 +
.../es_tools/index_replica_setter.py | 118 +
.../es_tools/index_shard_allocator.py | 148 ++
.../es_tools/mapping_deleter.py | 34 +
.../es_tools/mapping_retriever.py | 45 +
.../es_tools/monitor_tasks.py | 41 +
utils/usergrid-util-python/index_test/README.md | 1 +
.../index_test/document_creator.py | 254 ++
.../index_test/index_test_mixed_batch.py | 545 +++++
.../index_test/index_test_single_type_batch.py | 547 +++++
utils/usergrid-util-python/requirements.txt | 4 +
.../activity_streams/activity_streams.py | 132 ++
.../samples/beacon-event-example.py | 196 ++
.../samples/counter_test.py | 31 +
utils/usergrid-util-python/setup.py | 40 +
.../usergrid_tools/__init__.py | 4 +
.../usergrid_tools/general/__init__.py | 0
.../usergrid_tools/general/deleter.py | 151 ++
.../general/duplicate_name_checker.py | 25 +
.../usergrid_tools/general/queue_monitor.py | 119 +
.../usergrid_tools/general/url_tester.py | 87 +
.../general/user_confirm_activate.py | 29 +
.../usergrid_tools/general/user_creator.py | 49 +
.../usergrid_tools/groups/__init__.py | 0
.../usergrid_tools/groups/big_group_creater.py | 86 +
.../usergrid_tools/indexing/README.md | 22 +
.../usergrid_tools/indexing/__init__.py | 0
.../usergrid_tools/indexing/batch_index_test.py | 340 +++
.../indexing/entity_index_test.py | 317 +++
.../usergrid_tools/iterators/README.md | 8 +
.../usergrid_tools/iterators/__init__.py | 0
.../usergrid_tools/iterators/simple_iterator.py | 79 +
.../iterators/usergrid_cross_region_iterator.py | 409 ++++
.../usergrid_tools/library_check.py | 23 +
.../usergrid_tools/migration/README.md | 234 ++
.../usergrid_tools/migration/__init__.py | 2 +
.../migration/usergrid_data_exporter.py | 923 ++++++++
.../migration/usergrid_data_migrator.py | 2168 ++++++++++++++++++
.../usergrid_tools/parse_importer/README.md | 90 +
.../usergrid_tools/parse_importer/__init__.py | 0
.../parse_importer/parse_importer.py | 385 ++++
.../usergrid_tools/permissions/README.md | 3 +
.../usergrid_tools/permissions/permissions.py | 146 ++
.../usergrid_tools/queue/README.md | 1 +
.../queue/dlq-iterator-checker.py | 143 ++
.../usergrid_tools/queue/dlq_requeue.py | 173 ++
.../queue/queue-config-sample.json | 22 +
.../usergrid_tools/queue/queue_cleaner.py | 155 ++
.../usergrid_tools/redis/redis_iterator.py | 30 +
.../usergrid_tools/redis/redisscan.py | 15 +
76 files changed, 10631 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/.gitignore
----------------------------------------------------------------------
diff --git a/sdks/python/.gitignore b/sdks/python/.gitignore
new file mode 100644
index 0000000..ba74660
--- /dev/null
+++ b/sdks/python/.gitignore
@@ -0,0 +1,57 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/LICENSE
----------------------------------------------------------------------
diff --git a/sdks/python/LICENSE b/sdks/python/LICENSE
new file mode 100755
index 0000000..8f71f43
--- /dev/null
+++ b/sdks/python/LICENSE
@@ -0,0 +1,202 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/README.md
----------------------------------------------------------------------
diff --git a/sdks/python/README.md b/sdks/python/README.md
new file mode 100755
index 0000000..cc2af97
--- /dev/null
+++ b/sdks/python/README.md
@@ -0,0 +1,16 @@
+# Usergrid Python SDK
+
+# Overview
+This is a starter project for the Usergrid Python SDK. It is a work in progress.
+
+# Installation
+
+## PIP (http://pip.readthedocs.org/en/stable/installing/)
+
+`pip install usergrid`
+
+## Manual installation
+
+- `git clone git@github.com:jwest-apigee/usergrid-python.git`
+- `cd usergrid-python`
+- `pip install -e .`
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/sample_app.py
----------------------------------------------------------------------
diff --git a/sdks/python/sample_app.py b/sdks/python/sample_app.py
new file mode 100755
index 0000000..a829736
--- /dev/null
+++ b/sdks/python/sample_app.py
@@ -0,0 +1,68 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+from usergrid import Usergrid
+
+__author__ = 'ApigeeCorporation'
+
+
+def main():
+ Usergrid.init(org_id='jwest1',
+ app_id='sandbox')
+
+ response = Usergrid.DELETE('pets', 'max')
+ if not response.ok:
+ print 'Failed to delete max: %s' % response
+ exit()
+
+ response = Usergrid.DELETE('owners', 'jeff')
+ if not response.ok:
+ print 'Failed to delete Jeff: %s' % response
+ exit()
+
+ response = Usergrid.POST('pets', {'name': 'max'})
+
+ if response.ok:
+ pet = response.first()
+ print pet
+ response = Usergrid.POST('owners', {'name': 'jeff'})
+
+ if response.ok:
+ owner = response.first()
+ print owner
+ response = pet.connect('ownedBy', owner)
+
+ if response.ok:
+ print 'Connected!'
+
+ response = pet.disconnect('ownedBy', owner)
+
+ if response.ok:
+ print 'all done!'
+ else:
+ print response
+ else:
+ print 'failed to connect: %s' % response
+
+ else:
+ print 'Failed to create Jeff: %s' % response
+
+ else:
+ print response
+
+
+main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/setup.py
----------------------------------------------------------------------
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
new file mode 100755
index 0000000..6eec51f
--- /dev/null
+++ b/sdks/python/setup.py
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+from setuptools import setup, find_packages
+
+VERSION = '0.1.11'
+
+setup(
+ name='usergrid',
+ version=VERSION,
+ description='Usergrid SDK for Python',
+ url='http://usergrid.apache.org',
+ download_url="https://codeload.github.com/jwest-apigee/usergrid-python/zip/v" + VERSION,
+ author='Jeff West',
+ author_email='jwest@apigee.com',
+ packages=find_packages(),
+ install_requires=[
+ 'requests',
+ 'urllib3'
+ ],
+ entry_points={
+ }
+)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridApplication.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridApplication.py b/sdks/python/usergrid/UsergridApplication.py
new file mode 100644
index 0000000..cedd5b1
--- /dev/null
+++ b/sdks/python/usergrid/UsergridApplication.py
@@ -0,0 +1,62 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+import logging
+from usergrid import UsergridError, UsergridCollection
+from usergrid.app_templates import app_url_template
+
+
+class UsergridApplication(object):
+ def __init__(self, app_id, client):
+ self.app_id = app_id
+ self.client = client
+ self.logger = logging.getLogger('usergrid.UsergridClient')
+
+ def list_collections(self):
+ url = app_url_template.format(app_id=self.app_id,
+ **self.client.url_data)
+ r = self.client.get(url)
+
+ if r.status_code == 200:
+ api_response = r.json()
+ collection_list = api_response.get('entities')[0].get('metadata', {}).get('collections', {})
+ collections = {}
+
+ for collection_name in collection_list:
+ collections[collection_name] = UsergridCollection(self.client.org_id,
+ self.app_id,
+ collection_name,
+ self.client)
+
+ return collections
+
+ else:
+ raise UsergridError(message='Unable to post to list collections',
+ status_code=r.status_code,
+ api_response=r,
+ url=url)
+
+ def collection(self, collection_name):
+ return UsergridCollection(self.client.org_id,
+ self.app_id,
+ collection_name,
+ self.client)
+
+ def authenticate_app_client(self,
+ **kwargs):
+
+ return self.client.authenticate_app_client(self.app_id, **kwargs)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridAuth.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridAuth.py b/sdks/python/usergrid/UsergridAuth.py
new file mode 100644
index 0000000..3406312
--- /dev/null
+++ b/sdks/python/usergrid/UsergridAuth.py
@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+import json
+import requests
+from usergrid.management_templates import org_token_url_template
+
+
+class UsergridAuth:
+ def __init__(self,
+ grant_type,
+ url_template,
+ username=None,
+ password=None,
+ client_id=None,
+ client_secret=None,
+ token_ttl_seconds=86400):
+
+ self.grant_type = grant_type
+ self.username = username
+ self.password = password
+ self.client_id = client_id
+ self.client_secret = client_secret
+ self.token_ttl_seconds = token_ttl_seconds
+ self.url_template = url_template
+ self.access_token = None
+
+ def get_token_request(self):
+ if self.grant_type == 'client_credentials':
+ return {
+ 'grant_type': 'client_credentials',
+ 'client_id': self.client_id,
+ 'client_secret': self.client_secret,
+ 'ttl': self.token_ttl_seconds * 1000
+ }
+ elif self.grant_type == 'password':
+ return {
+ 'grant_type': 'password',
+ 'username': self.username,
+ 'password': self.password,
+ 'ttl': self.token_ttl_seconds * 1000
+ }
+
+ else:
+ raise ValueError('Unspecified/unknown grant type: %s' % self.grant_type)
+
+ def authenticate(self, client):
+ token_request = self.get_token_request()
+
+ url = self.url_template.format(**client.url_data)
+
+ r = requests.post(url, data=json.dumps(token_request))
+
+ if r.status_code == 200:
+ response = r.json()
+ self.access_token = response.get('access_token')
+
+ else:
+ raise ValueError('Unable to authenticate: %s' % r.text)
+
+
+class UsergridOrgAuth(UsergridAuth):
+ def __init__(self, client_id, client_secret, token_ttl_seconds=86400):
+ UsergridAuth.__init__(self,
+ grant_type='client_credentials',
+ url_template=org_token_url_template,
+ client_id=client_id,
+ client_secret=client_secret,
+ token_ttl_seconds=token_ttl_seconds)
+
+
+class UsergridAppAuth(UsergridAuth):
+ def __init__(self, client_id, client_secret, token_ttl_seconds=86400):
+ UsergridAuth.__init__(self,
+ grant_type='client_credentials',
+ url_template=app_token_url_template,
+ client_id=client_id,
+ client_secret=client_secret,
+ token_ttl_seconds=token_ttl_seconds)
+
+
+class UsergridUserAuth(UsergridAuth):
+ def __init__(self, username, password, token_ttl_seconds=86400):
+ UsergridAuth.__init__(self,
+ grant_type='password',
+ url_template=app_token_url_template,
+ username=username,
+ password=password,
+ token_ttl_seconds=token_ttl_seconds)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridClient.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridClient.py b/sdks/python/usergrid/UsergridClient.py
new file mode 100644
index 0000000..2ab8f73
--- /dev/null
+++ b/sdks/python/usergrid/UsergridClient.py
@@ -0,0 +1,398 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+import json
+import logging
+import requests
+from usergrid.UsergridAuth import UsergridAppAuth
+from usergrid.app_templates import get_entity_url_template, post_collection_url_template, put_entity_url_template, \
+ delete_entity_url_template, connect_entities_by_type_template, assign_role_url_template
+
+
+def value_error(message):
+ raise ValueError(message)
+
+
+def usergrid_error(r):
+ pass
+
+
+class Usergrid(object):
+ client = None
+
+ @staticmethod
+ def init(org_id,
+ app_id,
+ **kwargs):
+ Usergrid.client = UsergridClient(org_id, app_id, **kwargs)
+
+ @staticmethod
+ def GET(collection, uuid_name, **kwargs):
+ return Usergrid.client.GET(collection, uuid_name, **kwargs)
+
+ @staticmethod
+ def PUT(collection, uuid_name, data, **kwargs):
+ return Usergrid.client.PUT(collection, uuid_name, data, **kwargs)
+
+ @staticmethod
+ def POST(collection, data, **kwargs):
+ return Usergrid.client.POST(collection, data, **kwargs)
+
+ @staticmethod
+ def DELETE(collection, uuid_name, **kwargs):
+ return Usergrid.client.DELETE(collection, uuid_name, **kwargs)
+
+ @staticmethod
+ def connect_entities(from_entity, relationship, to_entity, **kwargs):
+ return Usergrid.client.connect_entities(from_entity, relationship, to_entity, **kwargs)
+
+ @staticmethod
+ def disconnect_entities(from_entity, relationship, to_entity, **kwargs):
+ return Usergrid.client.disconnect_entities(from_entity, relationship, to_entity, **kwargs)
+
+ @staticmethod
+ def assign_role(role_uuid_name, user_entity, **kwargs):
+ return Usergrid.client.assign_role(role_uuid_name, user_entity, **kwargs)
+
+
+class UsergridResponse(object):
+ def __init__(self, api_response, client):
+ self.api_response = api_response
+ self.client = client
+
+ if api_response is None:
+ self.ok = False
+ self.body = 'No Response'
+
+ else:
+ self.headers = api_response.headers
+
+ if api_response.status_code == 200:
+ self.ok = True
+ self.body = api_response.json()
+ self.entities = self.body.get('entities', [])
+
+ else:
+ self.ok = False
+
+ if api_response.headers.get('Content-type') == 'application/json':
+ self.body = api_response.json()
+ else:
+ self.body = 'HTTP %s: %s' % (api_response.status_code, api_response.text)
+
+ def __str__(self):
+ return json.dumps(self.body)
+
+ def first(self):
+ return UsergridEntity(entity_data=self.entities[0]) if self.ok and self.entities and len(
+ self.entities) > 0 else None
+
+ def entity(self):
+ return self.first()
+
+ def last(self):
+ return UsergridEntity(entity_data=self.entities[len(self.entities) - 1]) if self.ok and self.entities and len(
+ self.entities) > 0 else None
+
+ def has_next_page(self):
+ return 'cursor' in self.body if self.ok else False
+
+
+class UsergridEntity(object):
+ def __init__(self, entity_data):
+ self.entity_data = entity_data
+
+ def __str__(self):
+ return json.dumps(self.entity_data)
+
+ def get(self, name, default=None):
+ return self.entity_data.get(name, default)
+
+ def entity_id(self):
+
+ if self.entity_data.get('type', '').lower() in ['users', 'user']:
+ return self.entity_data.get('uuid', self.entity_data.get('username'))
+
+ return self.entity_data.get('uuid', self.entity_data.get('name'))
+
+ def can_mutate_or_load(self):
+ entity_id = self.entity_id()
+
+ if entity_id is None or self.entity_data.get('type') is None:
+ return False
+
+ return True
+
+ def put_property(self, name, value):
+ self.entity_data[name] = value
+
+ def put_properties(self, properties):
+ if isinstance(properties, dict):
+ self.entity_data.update(properties)
+
+ def remove_property(self, name):
+
+ if name is not None and name in self.entity_data:
+ del self.entity_data[name]
+
+ def remove_properties(self, properties):
+ if isinstance(properties, (list, dict)):
+ for property_name in properties:
+ self.remove_property(property_name)
+
+ def append(self, array_name, value):
+ if array_name in self.entity_data:
+ if isinstance(self.entity_data[array_name], list):
+ self.entity_data[array_name].append(value)
+ else:
+ self.entity_data[array_name] = [value]
+
+ def prepend(self, array_name, value):
+ if array_name in self.entity_data:
+ if isinstance(self.entity_data[array_name], list):
+ self.entity_data[array_name].pre(value)
+ else:
+ self.entity_data[array_name] = [value]
+
+ def insert(self, array_name, value, index):
+ if array_name in self.entity_data:
+ if isinstance(self.entity_data[array_name], list):
+ self.entity_data[array_name].insert(index, value)
+
+ def shift(self, array_name):
+ if array_name in self.entity_data:
+ if isinstance(self.entity_data[array_name], list):
+ value = self.entity_data[array_name][0]
+ self.entity_data[array_name] = self.entity_data[array_name][1:]
+ return value
+
+ return None
+
+ def reload(self):
+ if not self.can_mutate_or_load():
+ raise ValueError('Unable to reload entity: No uuid nor name')
+
+ response = Usergrid.GET(collection=self.entity_data.get('type'),
+ uuid_name=self.entity_id())
+ if response.ok:
+ self.entity_data.update(response.entity().entity_data)
+
+ else:
+ raise ValueError('Unable to reload entity: %s' % response)
+
+ def save(self):
+ if not self.can_mutate_or_load():
+ raise ValueError('Unable to save entity: No uuid nor name')
+
+ response = Usergrid.PUT(collection=self.entity_data.get('type'),
+ uuid_name=self.entity_id(),
+ data=self.entity_data)
+
+ if response.ok and 'uuid' not in self.entity_data:
+ self.entity_data['uuid'] = response.entity().get('uuid')
+
+ return response
+
+ def remove(self):
+ if not self.can_mutate_or_load():
+ raise ValueError('Unable to delete entity: No uuid nor name')
+
+ return Usergrid.DELETE(collection=self.entity_data.get('type'),
+ uuid_name=self.entity_id())
+
+ def get_connections(self, relationship, direction='connecting'):
+ pass
+
+ def connect(self, relationship, to_entity):
+
+ if not to_entity.can_mutate_or_load():
+ raise ValueError('Unable to connect to entity - no uuid or name')
+
+ if not self.can_mutate_or_load():
+ raise ValueError('Unable from connect to entity - no uuid or name')
+
+ return Usergrid.connect_entities(self, relationship, to_entity)
+
+ def disconnect(self, relationship, to_entity):
+ if not to_entity.can_mutate_or_load():
+ raise ValueError('Unable to connect to entity - no uuid or name')
+
+ if not self.can_mutate_or_load():
+ raise ValueError('Unable from connect to entity - no uuid or name')
+
+ return Usergrid.disconnect_entities(self, relationship, to_entity)
+
+ def attach_asset(self, filename, data, content_type):
+ pass
+
+ def download_asset(self, content_type=None):
+ pass
+
+
+class UsergridClient(object):
+ def __init__(self,
+ org_id,
+ app_id,
+ base_url='http://api.usergrid.com',
+ client_id=None,
+ client_secret=None,
+ token_ttl_seconds=86400,
+ auth_fallback="none"):
+
+ self.base_url = base_url
+ self.org_id = org_id
+ self.app_id = app_id
+ self.auth_fallback = auth_fallback
+ self.logger = logging.getLogger('usergrid.UsergridClient')
+ self.session = requests.Session()
+
+ self.url_data = {
+ 'base_url': base_url,
+ 'org_id': org_id,
+ 'app_id': app_id
+ }
+
+ if client_id and not client_secret:
+ value_error('Client ID Specified but not Secret')
+
+ elif client_secret and not client_id:
+ value_error('Client ID Specified but not Secret')
+
+ elif client_secret and client_id:
+ self.auth = UsergridAppAuth(client_id=client_id,
+ client_secret=client_secret,
+ token_ttl_seconds=token_ttl_seconds)
+
+ self.auth.authenticate(self)
+ self.session.headers.update({'Authorization': 'Bearer %s' % self.auth.access_token})
+
+ def __str__(self):
+ return json.dumps({
+ 'base_url': self.base_url,
+ 'org_id': self.org_id,
+ 'app_id': self.app_id,
+ 'access_token': self.auth.access_token
+ })
+
+ def GET(self, collection, uuid_name, connections='none', auth=None, **kwargs):
+ url = get_entity_url_template.format(collection=collection,
+ uuid_name=uuid_name,
+ connections=connections,
+ **self.url_data)
+ if auth:
+ r = requests.get(url, headers={'Authorization': 'Bearer %s' % auth.access_token})
+
+ else:
+ r = self.session.get(url)
+
+ return UsergridResponse(r, self)
+
+ def PUT(self, collection, uuid_name, data, auth=None, **kwargs):
+ url = put_entity_url_template.format(collection=collection,
+ uuid_name=uuid_name,
+ **self.url_data)
+
+ if auth:
+ r = requests.put(url,
+ data=json.dumps(data),
+ headers={'Authorization': 'Bearer %s' % auth.access_token})
+ else:
+ r = self.session.put(url, data=json.dumps(data))
+
+ return UsergridResponse(r, self)
+
+ def POST(self, collection, data, auth=None, **kwargs):
+ url = post_collection_url_template.format(collection=collection,
+ **self.url_data)
+
+ if auth:
+ r = requests.post(url,
+ data=json.dumps(data),
+ headers={'Authorization': 'Bearer %s' % auth.access_token})
+ else:
+ r = self.session.post(url, data=json.dumps(data))
+
+ return UsergridResponse(r, self)
+
+ def DELETE(self, collection, uuid_name, auth=None, **kwargs):
+ url = delete_entity_url_template.format(collection=collection,
+ uuid_name=uuid_name,
+ **self.url_data)
+
+ if auth:
+ r = requests.delete(url, headers={'Authorization': 'Bearer %s' % auth.access_token})
+ else:
+ r = self.session.delete(url)
+
+ return UsergridResponse(r, self)
+
+ def connect_entities(self, from_entity, relationship, to_entity, auth=None, **kwargs):
+
+ url = connect_entities_by_type_template.format(from_collection=from_entity.get('type'),
+ from_uuid_name=from_entity.entity_id(),
+ relationship=relationship,
+ to_collection=to_entity.get('type'),
+ to_uuid_name=to_entity.entity_id(),
+ **self.url_data)
+
+ if auth:
+ r = requests.post(url, headers={'Authorization': 'Bearer %s' % auth.access_token})
+ else:
+ r = self.session.post(url)
+
+ return UsergridResponse(r, self)
+
+ def assign_role(self, role_uuid_name, entity, auth=None, **kwargs):
+ url = assign_role_url_template.format(role_uuid_name=role_uuid_name,
+ entity_type=entity.get('type'),
+ entity_uuid_name=entity.entity_id(),
+ **self.url_data)
+
+ if auth:
+ r = requests.delete(url, headers={'Authorization': 'Bearer %s' % auth.access_token})
+ else:
+ r = self.session.delete(url)
+
+ return UsergridResponse(r, self)
+
+
+def disconnect_entities(self, from_entity, relationship, to_entity, auth=None, **kwargs):
+ url = connect_entities_by_type_template.format(from_collection=from_entity.get('type'),
+ from_uuid_name=from_entity.entity_id(),
+ relationship=relationship,
+ to_collection=to_entity.get('type'),
+ to_uuid_name=to_entity.entity_id(),
+ **self.url_data)
+
+ if auth:
+ r = requests.delete(url, headers={'Authorization': 'Bearer %s' % auth.access_token})
+ else:
+ r = self.session.delete(url)
+
+ return UsergridResponse(r, self)
+
+
+class UsergridUser(object):
+ def __init__(self):
+ pass
+
+
+class UsergridAsset(object):
+ def __init__(self, filename, data, content_type):
+ self.filename = filename
+ self.data = data
+ self.content_type = content_type
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridCollection.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridCollection.py b/sdks/python/usergrid/UsergridCollection.py
new file mode 100644
index 0000000..a37ad95
--- /dev/null
+++ b/sdks/python/usergrid/UsergridCollection.py
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+class UsergridCollection(object):
+ def __init__(self, org_id, app_id, collection_name, client):
+ self.org_id = org_id
+ self.app_id = app_id
+ self.collection_name = collection_name
+ self.client = client
+
+ def __str__(self):
+ return json.dumps({
+ 'org_id': self.org_id,
+ 'app_id': self.app_id,
+ 'collection_name': self.collection_name,
+ })
+
+ def entity(self, uuid):
+ pass
+
+ def entity_from_data(self, data):
+ return UsergridEntity(org_id=self.org_id,
+ app_id=self.app_id,
+ collection_name=self.collection_name,
+ data=data,
+ client=self.client)
+
+ def query(self, ql='select *', limit=100):
+ url = collection_query_url_template.format(app_id=self.app_id,
+ ql=ql,
+ limit=limit,
+ collection=self.collection_name,
+ **self.client.url_data)
+
+ return UsergridQuery(url, headers=self.client.headers)
+
+ def entities(self, **kwargs):
+ return self.query(**kwargs)
+
+ def post(self, entity, **kwargs):
+ url = collection_url_template.format(collection=self.collection_name,
+ app_id=self.app_id,
+ **self.client.url_data)
+
+ r = self.client.post(url, data=entity, **kwargs)
+
+ if r.status_code == 200:
+ api_response = r.json()
+ entity = api_response.get('entities')[0]
+ e = UsergridEntity(org_id=self.org_id,
+ app_id=self.app_id,
+ collection_name=self.collection_name,
+ data=entity,
+ client=self.client)
+ return e
+
+ else:
+ raise UsergridError(message='Unable to post to collection name=[%s]' % self.collection_name,
+ status_code=r.status_code,
+ data=entity,
+ api_response=r,
+ url=url)
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridConnection.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridConnection.py b/sdks/python/usergrid/UsergridConnection.py
new file mode 100644
index 0000000..82d3fdc
--- /dev/null
+++ b/sdks/python/usergrid/UsergridConnection.py
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+import logging
+
+
+class UsergridConnection(object):
+ def __init__(self, source_entity, verb, target_entity):
+ self.source_entity = source_entity
+ self.verb = verb
+ self.target_entity = target_entity
+ self.logger = logging.getLogger('usergrid.UsergridConnection')
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridError.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridError.py b/sdks/python/usergrid/UsergridError.py
new file mode 100644
index 0000000..a5cf0bb
--- /dev/null
+++ b/sdks/python/usergrid/UsergridError.py
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridOrganization.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridOrganization.py b/sdks/python/usergrid/UsergridOrganization.py
new file mode 100644
index 0000000..c0d345b
--- /dev/null
+++ b/sdks/python/usergrid/UsergridOrganization.py
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+from usergrid import UsergridApplication
+
+
+class UsergridOrganization(object):
+ def __init__(self, org_id, client):
+ self.org_id = org_id
+ self.client = client
+
+ def application(self, app_id):
+ return UsergridApplication(app_id, client=self.client)
+
+ def app(self, app_id):
+ return self.application(app_id)
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/UsergridQueryIterator.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridQueryIterator.py b/sdks/python/usergrid/UsergridQueryIterator.py
new file mode 100755
index 0000000..301ea7d
--- /dev/null
+++ b/sdks/python/usergrid/UsergridQueryIterator.py
@@ -0,0 +1,155 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+import json
+import logging
+import traceback
+import requests
+import time
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+
+class UsergridQueryIterator(object):
+ def __init__(self,
+ url,
+ operation='GET',
+ sleep_time=10,
+ page_delay=0,
+ headers=None,
+ data=None):
+
+ if not data:
+ data = {}
+ if not headers:
+ headers = {}
+
+ self.page_counter = 0
+ self.total_retrieved = 0
+ self.logger = logging.getLogger('usergrid.UsergridQuery')
+ self.data = data
+ self.headers = headers
+ self.url = url
+ self.operation = operation
+ self.next_cursor = None
+ self.entities = []
+ self.count_retrieved = 0
+ self._pos = 0
+ self.last_response = None
+ self.page_delay = page_delay
+ self.sleep_time = sleep_time
+ self.session = None
+
+ def _get_next_response(self, attempts=0):
+
+ if self.session is None:
+ self.session = requests.Session()
+
+ try:
+ if self.operation == 'PUT':
+ op = self.session.put
+ elif self.operation == 'DELETE':
+ op = self.session.delete
+ else:
+ op = self.session.get
+
+ target_url = self.url
+
+ if self.next_cursor is not None:
+ delim = '&' if '?' in target_url else '?'
+ target_url = '%s%scursor=%s' % (self.url, delim, self.next_cursor)
+
+ self.logger.debug('Operation=[%s] URL=[%s]' % (self.operation, target_url))
+
+ r = op(target_url, data=json.dumps(self.data), headers=self.headers)
+
+ if r.status_code == 200:
+ r_json = r.json()
+ count_retrieved = len(r_json.get('entities', []))
+ self.total_retrieved += count_retrieved
+ self.logger.debug('Retrieved [%s] entities in [%s]th page in [%s], total from [%s] is [%s]' % (
+ count_retrieved, self.page_counter, r.elapsed, self.url, self.total_retrieved))
+
+ return r_json
+
+ elif r.status_code in [401, 404] and 'service_resource_not_found' in r.text:
+ self.logger.error('Query Not Found [%s] on URL=[%s]: %s' % (r.status_code, target_url, r.text))
+ raise SystemError('Query Not Found [%s] on URL=[%s]: %s' % (r.status_code, target_url, r.text))
+
+ else:
+ if attempts < 10:
+ self.logger.error('Sleeping %s after HTTP [%s] for retry attempt=[%s] on URL=[%s], response: %s' % (
+ self.sleep_time, r.status_code, attempts, target_url, r.text))
+
+ time.sleep(self.sleep_time)
+
+ return self._get_next_response(attempts=attempts + 1)
+
+ else:
+ raise SystemError('Unable to get next response after %s attempts' % attempts)
+
+ except:
+ print traceback.format_exc()
+
+ def next(self):
+
+ if self.last_response is None:
+ self.logger.debug('getting first page, url=[%s]' % self.url)
+
+ self._process_next_page()
+
+ elif self._pos >= len(self.entities) > 0 and self.next_cursor is not None:
+
+ self.logger.debug('getting next page, count=[%s] url=[%s], cursor=[%s]' % (
+ self.count_retrieved, self.url, self.next_cursor))
+
+ self._process_next_page()
+ self.logger.debug('Sleeping [%s]s between pages' % self.page_delay)
+
+ time.sleep(self.page_delay)
+
+ if self._pos < len(self.entities):
+ response = self.entities[self._pos]
+ self._pos += 1
+ return response
+
+ raise StopIteration
+
+ def __iter__(self):
+ return self
+
+ def _process_next_page(self, attempts=0):
+
+ api_response = self._get_next_response()
+
+ if api_response is None:
+ message = 'Unable to retrieve query results from url=[%s]' % self.url
+ self.logger.error(message)
+ api_response = {}
+ raise StopIteration
+
+ self.last_response = api_response
+
+ self.entities = api_response.get('entities', [])
+ self.next_cursor = api_response.get('cursor', None)
+ self._pos = 0
+ self.count_retrieved += len(self.entities)
+ self.page_counter += 1
+
+ if self.next_cursor is None:
+ self.logger.debug('no cursor in response. Total pages=[%s], entities=[%s] url=[%s]' % (
+ self.page_counter, self.count_retrieved, self.url))
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/__init__.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/__init__.py b/sdks/python/usergrid/__init__.py
new file mode 100644
index 0000000..93f8273
--- /dev/null
+++ b/sdks/python/usergrid/__init__.py
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+__all__ = [
+ 'UsergridApplication',
+ 'UsergridClient',
+ 'UsergridConnection',
+ 'UsergridConnectionProfile',
+ 'UsergridEntity',
+ 'Usergrid',
+ 'UsergridError',
+ 'UsergridOrganization',
+ 'UsergridAuth',
+ 'UsergridQueryIterator',
+ 'UsergridResponse'
+]
+
+from .UsergridApplication import UsergridApplication
+from .UsergridClient import UsergridClient, Usergrid, UsergridResponse
+from .UsergridConnection import UsergridConnection
+from .UsergridOrganization import UsergridOrganization
+from .UsergridQueryIterator import UsergridQueryIterator
+from .UsergridAuth import UsergridAuth
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/app_templates.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/app_templates.py b/sdks/python/usergrid/app_templates.py
new file mode 100644
index 0000000..cb953ac
--- /dev/null
+++ b/sdks/python/usergrid/app_templates.py
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+org_url_template = "{base_url}/{org_id}"
+app_url_template = "%s/{app_id}" % org_url_template
+
+app_token_url_template = "%s/token" % app_url_template
+
+collection_url_template = "%s/{collection}" % app_url_template
+collection_query_url_template = "%s?ql={ql}&limit={limit}" % collection_url_template
+
+post_collection_url_template = collection_url_template
+entity_url_template = "%s/{uuid_name}" % collection_url_template
+get_entity_url_template = "%s?connections={connections}" % entity_url_template
+put_entity_url_template = entity_url_template
+delete_entity_url_template = entity_url_template
+
+assign_role_url_template = '%s/roles/{role_uuid_name}/{entity_type}/{entity_uuid_name}' % app_url_template
+
+connect_entities_by_type_template = '%s/{from_collection}/{from_uuid_name}/{relationship}/{to_collection}/{to_uuid_name}' % app_url_template
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/sdks/python/usergrid/management_templates.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/management_templates.py b/sdks/python/usergrid/management_templates.py
new file mode 100644
index 0000000..62c531c
--- /dev/null
+++ b/sdks/python/usergrid/management_templates.py
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. The ASF licenses this file to You
+# under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. For additional information regarding
+# copyright in this work, please see the NOTICE file in the top level
+# directory of this distribution.
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+management_base_url = '{base_url}/management'
+management_org_url_template = "%s/organizations/{org_id}" % management_base_url
+management_org_list_apps_url_template = "%s/applications" % management_org_url_template
+management_app_url_template = "%s/applications/{app_id}" % management_org_url_template
+
+org_token_url_template = "%s/token" % management_base_url
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/.gitignore
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/.gitignore b/utils/usergrid-util-python/.gitignore
new file mode 100644
index 0000000..a6e3315
--- /dev/null
+++ b/utils/usergrid-util-python/.gitignore
@@ -0,0 +1,61 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# custom
+sandbox
+0e4b82c5-9aad-45de-810a-ff07c281ed2d_1454177649_export.zip
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/LICENSE
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/LICENSE b/utils/usergrid-util-python/LICENSE
new file mode 100644
index 0000000..4db993e
--- /dev/null
+++ b/utils/usergrid-util-python/LICENSE
@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Jeffrey West
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/README.md
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/README.md b/utils/usergrid-util-python/README.md
new file mode 100644
index 0000000..7d3b533
--- /dev/null
+++ b/utils/usergrid-util-python/README.md
@@ -0,0 +1,15 @@
+# Usergrid Tools (in Python)
+
+## Prerequisites
+
+* Install the Usergrid Python SDK: `pip install usergrid`
+* Install Usergrid Tools: `pip install usergrid-tools`
+
+
+## Overview
+The purpose of this module is to provide tools for working with Usergrid. The tools included as console scripts are:
+* `usergrid_data_migrator` - [README](https://github.com/jwest-apigee/usergrid-util-python/blob/master/usergrid_tools/migration/README.md) A tool for migrating data from one Usergrid installation to another (or org1->org2)
+* `parse_importer` - [README](https://github.com/jwest-apigee/usergrid-util-python/blob/master/usergrid_tools/parse_importer/README.md) A tool for importing data from a Parse.com data export into Usergrid
+* `index_test` - [README](https://github.com/jwest-apigee/usergrid-util-python/blob/master/usergrid_tools/indexing/README.md) A tool for testing indexing latency in Usergrid
+
+For information on those tools, please see the respective README files
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/alias_mover.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/alias_mover.py b/utils/usergrid-util-python/es_tools/alias_mover.py
new file mode 100644
index 0000000..2a8fe02
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/alias_mover.py
@@ -0,0 +1,129 @@
+import json
+
+import requests
+
+example_request = {
+ "actions": [
+ {
+ "remove": {
+ "index": "apigee-vfmplus",
+ "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias"
+ }
+ },
+ {
+ "remove": {
+ "index": "apigee-vfmplus",
+ "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias"
+ }
+ },
+ {
+ "remove": {
+ "index": "apigee-vfmplus",
+ "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias"
+ }
+ },
+ {
+ "remove": {
+ "index": "apigee-vfmplus",
+ "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias"
+ }
+ },
+ {
+ "remove": {
+ "index": "apigee-vfmplus",
+ "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias"
+ }
+ },
+ {
+ "remove": {
+ "index": "apigee-vfmplus",
+ "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias"
+ }
+ },
+ {
+ "add": {
+ "index": "apigee-vmplus-docvalues",
+ "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias"
+ }
+ },
+ {
+ "add": {
+ "index": "apigee-vmplus-docvalues",
+ "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias"
+ }
+ },
+ {
+ "add": {
+ "index": "apigee-vmplus-docvalues",
+ "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias"
+ }
+ },
+ {
+ "add": {
+ "index": "apigee-vmplus-docvalues",
+ "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias"
+ }
+ },
+ {
+ "add": {
+ "index": "apigee-vmplus-docvalues",
+ "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias"
+ }
+ },
+ {
+ "add": {
+ "index": "apigee-vmplus-docvalues",
+ "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias"
+ }
+ }
+ ]
+}
+
+cluster = 'rug000sr_euwi'
+
+work = {
+ # 'remove': {
+ # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'rug000sr_euwi_applications_3',
+ # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'rug000sr_euwi_applications_3',
+ # },
+ 'add': {
+ '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'apigee-vfmplus-1-no-doc-18',
+ '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'apigee-vfmplus-1-no-doc-18',
+ }
+}
+
+actions = []
+
+for app_id, index in work.get('remove', {}).iteritems():
+ actions.append({
+ "remove": {
+ "index": index,
+ "alias": "%s_%s_read_alias" % (cluster, app_id)
+ },
+ })
+ actions.append({
+ "remove": {
+ "index": index,
+ "alias": "%s_%s_write_alias" % (cluster, app_id)
+ },
+ })
+
+for app_id, index in work['add'].iteritems():
+ actions.append({
+ "add": {
+ "index": index,
+ "alias": "%s_%s_read_alias" % (cluster, app_id)
+ },
+ })
+ actions.append({
+ "add": {
+ "index": index,
+ "alias": "%s_%s_write_alias" % (cluster, app_id)
+ },
+ })
+
+url = 'http://localhost:9200/_aliases'
+
+r = requests.post(url, data=json.dumps({'actions': actions}))
+
+print '%s: %s' % (r.status_code, r.text)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
new file mode 100644
index 0000000..a462124
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
@@ -0,0 +1,89 @@
+import json
+import time
+import requests
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+# The purpose of this script is to set certain nodes in an ElasticSearch cluster to be excluded from index allocation,
+# generally for the purpose of shutting down or restarting the node
+
+SHUTDOWN_NODES = True
+
+nodes = [
+ # 'res206wo',
+ # 'res207wo',
+]
+
+base_url = 'http://localhost:9200'
+
+exclude_nodes = nodes
+
+nodes_string = ",".join(exclude_nodes)
+
+print 'Excluding: ' + nodes_string
+url_template = '%s/_cluster/settings' % base_url
+
+status_code = 503
+
+while status_code >= 500:
+ r = requests.put(
+ '%s/_cluster/settings' % base_url,
+ data=json.dumps({
+ "transient": {
+ "cluster.routing.allocation.exclude._host": nodes_string
+ }
+ }))
+
+ status_code = r.status_code
+
+ print '%s: %s' % (r.status_code, r.text)
+
+ready = False
+
+nodes_shut_down = []
+
+while not ready:
+
+ ready = True
+ nodes_left = 0
+ bytes_left = 0
+
+ for node in exclude_nodes:
+ node_url = '%s/_nodes/%s/stats' % (base_url, node)
+ r = requests.get(node_url)
+
+ if r.status_code == 200:
+ # print r.text
+
+ node_stats = r.json()
+
+ for field, data in node_stats.get('nodes').iteritems():
+ if data.get('name') == node:
+ size = data.get('indices', {}).get('store', {}).get('size_in_bytes', 1)
+ docs = data.get('indices', {}).get('docs', {}).get('count', 1)
+
+ if size > 0 and docs > 0:
+ print 'Node: %s - size %s' % (node, size)
+ bytes_left += size
+ ready = False and ready
+ nodes_left += 1
+ else:
+ if SHUTDOWN_NODES:
+ if not node in nodes_shut_down:
+ nodes_shut_down.append(node)
+ shutdown_url = '%s/_cluster/nodes/%s/_shutdown' % (base_url, node)
+
+ print 'Shutting down node %s: %s' % (node, shutdown_url)
+
+ r = requests.post(shutdown_url)
+
+ if r.status_code == 200:
+ nodes_shut_down.append(node)
+ print 'Shut down node %s' % node
+ else:
+ print 'Shutdown failed: %s: %s' % (r.status_code, r.text)
+ if not ready:
+ print 'NOT READY! Waiting for %s nodes and %s GB' % (nodes_left, bytes_left / 1024.0 / 1000000)
+ time.sleep(10)
+
+# print 'READY TO MOVE!'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/command_sender.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/command_sender.py b/utils/usergrid-util-python/es_tools/command_sender.py
new file mode 100644
index 0000000..92ecfff
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/command_sender.py
@@ -0,0 +1,42 @@
+import json
+import requests
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+
+# Simple utility to send commands, useful to not have to recall the proper format
+
+#
+# url = 'http://localhost:9200/_cat/shards'
+#
+# r = requests.get(url)
+#
+# response = r.text
+#
+# print response
+
+data = {
+ "commands": [
+ {
+ "move": {
+ "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
+ "shard": 14,
+ "from_node": "res018sy",
+ "to_node": "res021sy"
+ }
+ },
+ {
+ "move": {
+ "index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
+ "shard": 12,
+ "from_node": "res018sy",
+ "to_node": "res009sy"
+ }
+ },
+
+ ]
+}
+
+r = requests.post('http://localhost:9211/_cluster/reroute', data=json.dumps(data))
+
+print r.text
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
new file mode 100644
index 0000000..f151fcb
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
@@ -0,0 +1,107 @@
+import json
+import re
+from multiprocessing.pool import Pool
+import requests
+
+# This script iterates an index and issues a PUT request for an empty string to force a reindex of the entity
+
+index_url_template = 'http://res013wo:9200/{index_name}/_search?size={size}&from={from_var}'
+
+index_names = [
+ 'es-index-name'
+]
+
+baas_url = 'http://localhost:8080/org/{app_id}/{collection}/{entity_id}'
+
+counter = 0
+size = 1000
+
+total_docs = 167501577
+from_var = 0
+page = 0
+
+work_items = []
+
+
+def work(item):
+ url = 'http://localhost:8080/org/{app_id}/{collection}/{entity_id}'.format(
+ app_id=item[0],
+ collection=item[1],
+ entity_id=item[2]
+ )
+
+ r_put = requests.put(url, data=json.dumps({'russo': ''}))
+
+ if r_put.status_code == 200:
+ print '[%s]: %s' % (r_put.status_code, url)
+
+ elif r_put.status_code:
+ print '[%s]: %s | %s' % (r_put.status_code, url, r.text)
+
+
+while from_var < total_docs:
+
+ from_var = page * size
+ page += 1
+
+ for index_name in index_names:
+
+ index_url = index_url_template.format(index_name=index_name, size=size, from_var=from_var)
+
+ print 'Getting URL: ' + index_url
+
+ r = requests.get(index_url)
+
+ if r.status_code != 200:
+ print r.text
+ exit()
+
+ response = r.json()
+
+ hits = response.get('hits', {}).get('hits')
+
+ re_app_id = re.compile('appId\((.+),')
+ re_ent_id = re.compile('entityId\((.+),')
+ re_type = re.compile('entityId\(.+,(.+)\)')
+
+ print 'Index: %s | hits: %s' % (index_name, len(hits))
+
+ for hit_data in hits:
+ source = hit_data.get('_source')
+
+ application_id = source.get('applicationId')
+
+ app_id_find = re_app_id.findall(application_id)
+
+ if len(app_id_find) > 0:
+ app_id = app_id_find[0]
+
+ if app_id != '5f20f423-f2a8-11e4-a478-12a5923b55dc':
+ continue
+
+ entity_id_tmp = source.get('entityId')
+
+ entity_id_find = re_ent_id.findall(entity_id_tmp)
+ entity_type_find = re_type.findall(entity_id_tmp)
+
+ if len(entity_id_find) > 0 and len(entity_type_find) > 0:
+ entity_id = entity_id_find[0]
+ collection = entity_type_find[0]
+
+ if collection in ['logs', 'log']:
+ print 'skipping logs...'
+ continue
+
+ work_items.append((app_id, collection, entity_id))
+
+ counter += 1
+
+pool = Pool(16)
+
+print 'Work Items: %s' % len(work_items)
+
+print 'Starting Work'
+
+pool.map(work, work_items)
+
+print 'done: %s' % counter
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/es_searcher.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_searcher.py b/utils/usergrid-util-python/es_tools/es_searcher.py
new file mode 100644
index 0000000..55e54ef
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/es_searcher.py
@@ -0,0 +1,24 @@
+import json
+import requests
+
+# Simple example of searching for a specific entity in ES
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+url_template = 'http://localhost:9200/pea000ug_applications_2/_search'
+
+request = {
+ "query": {
+ "term": {
+ "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,constratus)"
+ }
+ }
+}
+
+# url_template = 'http://localhost:9200/_search'
+# r = requests.get(url)
+r = requests.get(url_template, data=json.dumps(request))
+
+print r.status_code
+print json.dumps(r.json(), indent=2)
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/es_tools/index_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_deleter.py b/utils/usergrid-util-python/es_tools/index_deleter.py
new file mode 100644
index 0000000..a697cf8
--- /dev/null
+++ b/utils/usergrid-util-python/es_tools/index_deleter.py
@@ -0,0 +1,68 @@
+import requests
+import logging
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+
+# utility for deleting indexes that are no longer needed. Given:
+# A) a set of strings to include when evaluating the index names to delete
+# B) a set of strings to Exclude when evaluating the index names to delete
+#
+# The general logic is:
+# 1) If the include set is empty, or if the index name contains a string in the 'include' set, then delete
+# 2) If the index contains a string in the exclude list, do not delete
+
+url_base = 'http://localhost:9200'
+
+r = requests.get(url_base + "/_stats")
+
+indices = r.json()['indices']
+
+print 'retrieved %s indices' % len(indices)
+
+NUMBER_VALUE = 0
+
+includes = [
+ 'rug002sr_euwi',
+ # 'rug002mr',
+ # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3',
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
+]
+
+excludes = [
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c8',
+ # 'b6768a08-b5d5-11e3-a495-10ddb1de66c3',
+ # 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
+ # 'a34ad389-b626-11e4-848f-06b49118d7d0'
+]
+
+counter = 0
+process = False
+delete_counter = 0
+
+for index in indices:
+ process = False
+ counter += 1
+
+ print 'index %s of %s' % (counter, len(indices))
+
+ if len(includes) == 0:
+ process = True
+ else:
+ for include in includes:
+
+ if include in index:
+ process = True
+
+ if len(excludes) > 0:
+ for exclude in excludes:
+ if exclude in index:
+ process = False
+
+ if process:
+ delete_counter += 1
+
+ url_template = '%s/%s' % (url_base, index)
+ print 'DELETING Index [%s] %s at URL %s' % (delete_counter, index, url_template)
+
+ response = requests.delete('%s/%s' % (url_base, index))
[43/50] [abbrv] usergrid git commit: updated Apache license header
Posted by mr...@apache.org.
updated Apache license header
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3b9e3c46
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3b9e3c46
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3b9e3c46
Branch: refs/heads/master
Commit: 3b9e3c4645473c12e4627bea181d9e2c329fda18
Parents: c212f1f
Author: Jeff West <jw...@apigee.com>
Authored: Thu Jul 28 14:09:34 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Thu Jul 28 14:09:34 2016 -0700
----------------------------------------------------------------------
sdks/python/GUIDE.md | 2 +
sdks/python/README.rst | 20 ++++++
sdks/python/sample_app.py | 43 ++++++++-----
sdks/python/setup.py | 74 +++++++++++++---------
sdks/python/usergrid/UsergridApplication.py | 34 +++++-----
sdks/python/usergrid/UsergridAuth.py | 34 +++++-----
sdks/python/usergrid/UsergridClient.py | 34 +++++-----
sdks/python/usergrid/UsergridCollection.py | 34 +++++-----
sdks/python/usergrid/UsergridConnection.py | 34 +++++-----
sdks/python/usergrid/UsergridError.py | 35 +++++-----
sdks/python/usergrid/UsergridOrganization.py | 34 +++++-----
sdks/python/usergrid/UsergridQueryIterator.py | 36 ++++++-----
sdks/python/usergrid/app_templates.py | 36 ++++++-----
sdks/python/usergrid/management_templates.py | 36 ++++++-----
14 files changed, 274 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/GUIDE.md
----------------------------------------------------------------------
diff --git a/sdks/python/GUIDE.md b/sdks/python/GUIDE.md
new file mode 100644
index 0000000..0719005
--- /dev/null
+++ b/sdks/python/GUIDE.md
@@ -0,0 +1,2 @@
+
+https://docs.python.org/2/distutils/packageindex.html
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/README.rst
----------------------------------------------------------------------
diff --git a/sdks/python/README.rst b/sdks/python/README.rst
new file mode 100755
index 0000000..659384c
--- /dev/null
+++ b/sdks/python/README.rst
@@ -0,0 +1,20 @@
+**********
+Overview
+**********
+
+This is a starter project for the Usergrid Python SDK. It is a work in progress.
+
+**************************
+Installation
+**************************
+
+================================================
+Installation From Pypi Using PIP
+================================================
+
+PIP is a package manager for Python. For more information please view the information here: `PIP Installation Guide <http://pip.readthedocs.org/en/stable/installing/>`_
+
+From the command line::
+
+ pip install usergrid
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/sample_app.py
----------------------------------------------------------------------
diff --git a/sdks/python/sample_app.py b/sdks/python/sample_app.py
index a829736..9deefbe 100755
--- a/sdks/python/sample_app.py
+++ b/sdks/python/sample_app.py
@@ -1,23 +1,25 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
from usergrid import Usergrid
-__author__ = 'ApigeeCorporation'
+__author__ = 'Jeff West @ ApigeeCorporation'
def main():
@@ -25,11 +27,13 @@ def main():
app_id='sandbox')
response = Usergrid.DELETE('pets', 'max')
+
if not response.ok:
print 'Failed to delete max: %s' % response
exit()
response = Usergrid.DELETE('owners', 'jeff')
+
if not response.ok:
print 'Failed to delete Jeff: %s' % response
exit()
@@ -38,12 +42,16 @@ def main():
if response.ok:
pet = response.first()
+
print pet
+
response = Usergrid.POST('owners', {'name': 'jeff'})
if response.ok:
owner = response.first()
+
print owner
+
response = pet.connect('ownedBy', owner)
if response.ok:
@@ -55,6 +63,7 @@ def main():
print 'all done!'
else:
print response
+
else:
print 'failed to connect: %s' % response
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/setup.py
----------------------------------------------------------------------
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 6eec51f..8a2d332 100755
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -1,39 +1,51 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
__author__ = 'Jeff West @ ApigeeCorporation'
from setuptools import setup, find_packages
-VERSION = '0.1.11'
+VERSION = '0.1.13.1'
+
+with open('README.rst') as file:
+ long_description = file.read()
setup(
- name='usergrid',
- version=VERSION,
- description='Usergrid SDK for Python',
- url='http://usergrid.apache.org',
- download_url="https://codeload.github.com/jwest-apigee/usergrid-python/zip/v" + VERSION,
- author='Jeff West',
- author_email='jwest@apigee.com',
- packages=find_packages(),
- install_requires=[
- 'requests',
- 'urllib3'
- ],
- entry_points={
- }
+ name='usergrid',
+ version=VERSION,
+ description='Usergrid SDK for Python',
+ url='http://usergrid.apache.org',
+ download_url="https://codeload.github.com/jwest-apigee/usergrid-python/zip/v" + VERSION,
+ author='Jeff West',
+ author_email='jwest@apigee.com',
+ packages=find_packages(),
+ long_description=long_description,
+ install_requires=[
+ 'requests',
+ 'urllib3'
+ ],
+ entry_points={
+ },
+ classifiers=[
+ 'Development Status :: 4 - Beta',
+ 'Intended Audience :: Developers',
+ 'Operating System :: OS Independent',
+ 'Topic :: Software Development',
+ ]
)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/UsergridApplication.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridApplication.py b/sdks/python/usergrid/UsergridApplication.py
index cedd5b1..f11f54d 100644
--- a/sdks/python/usergrid/UsergridApplication.py
+++ b/sdks/python/usergrid/UsergridApplication.py
@@ -1,19 +1,21 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
import logging
from usergrid import UsergridError, UsergridCollection
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/UsergridAuth.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridAuth.py b/sdks/python/usergrid/UsergridAuth.py
index 3406312..f29bcb9 100644
--- a/sdks/python/usergrid/UsergridAuth.py
+++ b/sdks/python/usergrid/UsergridAuth.py
@@ -1,19 +1,21 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/UsergridClient.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridClient.py b/sdks/python/usergrid/UsergridClient.py
index e4b8ae1..d204288 100644
--- a/sdks/python/usergrid/UsergridClient.py
+++ b/sdks/python/usergrid/UsergridClient.py
@@ -1,19 +1,21 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
import json
import logging
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/UsergridCollection.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridCollection.py b/sdks/python/usergrid/UsergridCollection.py
index a37ad95..a3d3020 100644
--- a/sdks/python/usergrid/UsergridCollection.py
+++ b/sdks/python/usergrid/UsergridCollection.py
@@ -1,19 +1,21 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
class UsergridCollection(object):
def __init__(self, org_id, app_id, collection_name, client):
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/UsergridConnection.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridConnection.py b/sdks/python/usergrid/UsergridConnection.py
index 82d3fdc..83836fd 100644
--- a/sdks/python/usergrid/UsergridConnection.py
+++ b/sdks/python/usergrid/UsergridConnection.py
@@ -1,19 +1,21 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
import logging
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/UsergridError.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridError.py b/sdks/python/usergrid/UsergridError.py
index a5cf0bb..99e3507 100644
--- a/sdks/python/usergrid/UsergridError.py
+++ b/sdks/python/usergrid/UsergridError.py
@@ -1,17 +1,18 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
-
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/UsergridOrganization.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridOrganization.py b/sdks/python/usergrid/UsergridOrganization.py
index c0d345b..3ee88d2 100644
--- a/sdks/python/usergrid/UsergridOrganization.py
+++ b/sdks/python/usergrid/UsergridOrganization.py
@@ -1,19 +1,21 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
from usergrid import UsergridApplication
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/UsergridQueryIterator.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridQueryIterator.py b/sdks/python/usergrid/UsergridQueryIterator.py
index 301ea7d..e487fb3 100755
--- a/sdks/python/usergrid/UsergridQueryIterator.py
+++ b/sdks/python/usergrid/UsergridQueryIterator.py
@@ -1,19 +1,21 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
import json
import logging
@@ -21,7 +23,7 @@ import traceback
import requests
import time
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
class UsergridQueryIterator(object):
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/app_templates.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/app_templates.py b/sdks/python/usergrid/app_templates.py
index cb953ac..3598587 100644
--- a/sdks/python/usergrid/app_templates.py
+++ b/sdks/python/usergrid/app_templates.py
@@ -1,21 +1,23 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
org_url_template = "{base_url}/{org_id}"
app_url_template = "%s/{app_id}" % org_url_template
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3b9e3c46/sdks/python/usergrid/management_templates.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/management_templates.py b/sdks/python/usergrid/management_templates.py
index 62c531c..c231b49 100644
--- a/sdks/python/usergrid/management_templates.py
+++ b/sdks/python/usergrid/management_templates.py
@@ -1,21 +1,23 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. The ASF licenses this file to You
-# under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. For additional information regarding
-# copyright in this work, please see the NOTICE file in the top level
-# directory of this distribution.
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
-__author__ = 'Jeff West @ ApigeeCorporation'
+__author__ = 'Jeff.West@yahoo.com'
management_base_url = '{base_url}/management'
management_org_url_template = "%s/organizations/{org_id}" % management_base_url
[40/50] [abbrv] usergrid git commit: Merge branch 'master' into
apigee-sso-provider
Posted by mr...@apache.org.
Merge branch 'master' into apigee-sso-provider
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/50eee863
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/50eee863
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/50eee863
Branch: refs/heads/master
Commit: 50eee86310261612a9bcb6ada748a281094bddca
Parents: ddde8dd 055289a
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 27 16:52:51 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 27 16:52:51 2016 -0700
----------------------------------------------------------------------
.../apache/usergrid/java/client/UsergridClient.java | 2 ++
.../java/client/UsergridRequestManager.java | 13 ++++++++++++-
.../usergrid/java/client/auth/UsergridAuth.java | 1 +
.../usergrid/java/client/auth/UsergridUserAuth.java | 16 ++++++++++++++--
.../usergrid/java/client/query/UsergridQuery.java | 5 -----
.../usergrid/query/validator/ApiServerRunner.java | 2 +-
6 files changed, 30 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[49/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/547/head' of github.com:apache/usergrid
Posted by mr...@apache.org.
Merge commit 'refs/pull/547/head' of github.com:apache/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f299c90c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f299c90c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f299c90c
Branch: refs/heads/master
Commit: f299c90c381917cfdcb019af7e2f78f38b8b553c
Parents: a35cd38 98f95f9
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Aug 1 09:52:22 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Aug 1 09:52:22 2016 -0700
----------------------------------------------------------------------
stack/README.md | 2 +-
.../main/resources/usergrid-default.properties | 35 ++-
stack/rest/pom.xml | 8 +
.../usergrid/rest/AbstractContextResource.java | 16 +
.../usergrid/rest/exceptions/AuthErrorInfo.java | 1 +
.../rest/management/ManagementResource.java | 164 +++++++++--
.../organizations/OrganizationsResource.java | 18 +-
.../organizations/users/UsersResource.java | 14 +-
.../rest/management/users/UserResource.java | 68 +++--
.../rest/management/users/UsersResource.java | 19 +-
.../shiro/filters/BasicAuthSecurityFilter.java | 3 +
.../ClientCredentialsSecurityFilter.java | 4 +
.../OAuth2AccessTokenSecurityFilter.java | 10 +-
.../security/shiro/filters/SecurityFilter.java | 12 +
.../rest/management/ExternalSSOEnabledIT.java | 175 +++++++++++
.../rest/management/ManagementResourceIT.java | 34 ++-
stack/services/pom.xml | 7 +
.../cassandra/ManagementServiceImpl.java | 36 ++-
...alSSOProviderAdminUserNotFoundException.java | 28 ++
.../security/shiro/utils/SubjectUtils.java | 32 +-
.../security/sso/ApigeeSSO2Provider.java | 215 ++++++++++++++
.../security/sso/ExternalSSOProvider.java | 42 +++
.../security/sso/SSOProviderFactory.java | 85 ++++++
.../security/sso/UsergridExternalProvider.java | 291 +++++++++++++++++++
.../usergrid/security/tokens/TokenService.java | 10 +-
.../tokens/cassandra/TokenServiceImpl.java | 281 +++++-------------
.../resources/usergrid-services-context.xml | 14 +
27 files changed, 1297 insertions(+), 327 deletions(-)
----------------------------------------------------------------------
[02/50] [abbrv] usergrid git commit: Add license header to new files.
Posted by mr...@apache.org.
Add license header to new files.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/71cbd8e5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/71cbd8e5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/71cbd8e5
Branch: refs/heads/master
Commit: 71cbd8e59499ea18c91c2fbaf3343d82652bb116
Parents: 6c36882
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jun 24 17:14:20 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jun 24 17:14:20 2016 -0700
----------------------------------------------------------------------
.../cassandra/ManagementServiceImpl.java | 7 +++++++
.../usergrid/security/sso/ApigeeSSO2Provider.java | 16 ++++++++++++++++
.../usergrid/security/sso/ExternalSSOProvider.java | 16 ++++++++++++++++
.../usergrid/security/sso/SSOProviderFactory.java | 17 ++++++++++++++++-
.../security/sso/UsergridExternalProvider.java | 16 ++++++++++++++++
5 files changed, 71 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/71cbd8e5/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index b56f211..b062f84 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -68,6 +68,7 @@ import org.apache.usergrid.security.shiro.utils.SubjectUtils;
import org.apache.usergrid.security.tokens.TokenCategory;
import org.apache.usergrid.security.tokens.TokenInfo;
import org.apache.usergrid.security.tokens.TokenService;
+import org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl;
import org.apache.usergrid.security.tokens.exceptions.TokenException;
import org.apache.usergrid.services.*;
import org.apache.usergrid.utils.*;
@@ -1525,6 +1526,12 @@ public class ManagementServiceImpl implements ManagementService {
@Override
public String getAccessTokenForAdminUser( UUID userId, long duration ) throws Exception {
+
+ if( properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_SSO_ENABLED).equalsIgnoreCase("true")){
+ throw new RuntimeException("SSO Integration is enabled, Admin users must login via provider: "+
+ properties.getProperty(TokenServiceImpl.USERGRID_EXTERNAL_PROVIDER));
+ }
+
return getTokenForPrincipal( ACCESS, null, smf.getManagementAppId(), ADMIN_USER, userId, duration );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/71cbd8e5/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
index 5400212..f35e546 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.security.sso;
import io.jsonwebtoken.*;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/71cbd8e5/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java
index eee46c6..180a675 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/ExternalSSOProvider.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.security.sso;
import org.apache.usergrid.management.UserInfo;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/71cbd8e5/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java
index fae0f7f..a3016c8 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/SSOProviderFactory.java
@@ -1,7 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.security.sso;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.security.tokens.cassandra.TokenServiceImpl;
import org.springframework.beans.factory.annotation.Autowired;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/71cbd8e5/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java
index a8d250d..a2e5fb2 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/UsergridExternalProvider.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.usergrid.security.sso;
import com.codahale.metrics.Counter;
[38/50] [abbrv] usergrid git commit: fixed indentation on
disconnect_entities
Posted by mr...@apache.org.
fixed indentation on disconnect_entities
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/04a896e7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/04a896e7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/04a896e7
Branch: refs/heads/master
Commit: 04a896e75dc802152c9331cfee7be07cb0b74b1b
Parents: 3bc3d78
Author: Jeff West <jw...@apigee.com>
Authored: Tue Jul 26 17:00:10 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Tue Jul 26 17:00:10 2016 -0700
----------------------------------------------------------------------
sdks/python/usergrid/UsergridClient.py | 27 +++++++++++++--------------
1 file changed, 13 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/04a896e7/sdks/python/usergrid/UsergridClient.py
----------------------------------------------------------------------
diff --git a/sdks/python/usergrid/UsergridClient.py b/sdks/python/usergrid/UsergridClient.py
index 2ab8f73..e4b8ae1 100644
--- a/sdks/python/usergrid/UsergridClient.py
+++ b/sdks/python/usergrid/UsergridClient.py
@@ -369,21 +369,20 @@ class UsergridClient(object):
return UsergridResponse(r, self)
+ def disconnect_entities(self, from_entity, relationship, to_entity, auth=None, **kwargs):
+ url = connect_entities_by_type_template.format(from_collection=from_entity.get('type'),
+ from_uuid_name=from_entity.entity_id(),
+ relationship=relationship,
+ to_collection=to_entity.get('type'),
+ to_uuid_name=to_entity.entity_id(),
+ **self.url_data)
+
+ if auth:
+ r = requests.delete(url, headers={'Authorization': 'Bearer %s' % auth.access_token})
+ else:
+ r = self.session.delete(url)
-def disconnect_entities(self, from_entity, relationship, to_entity, auth=None, **kwargs):
- url = connect_entities_by_type_template.format(from_collection=from_entity.get('type'),
- from_uuid_name=from_entity.entity_id(),
- relationship=relationship,
- to_collection=to_entity.get('type'),
- to_uuid_name=to_entity.entity_id(),
- **self.url_data)
-
- if auth:
- r = requests.delete(url, headers={'Authorization': 'Bearer %s' % auth.access_token})
- else:
- r = self.session.delete(url)
-
- return UsergridResponse(r, self)
+ return UsergridResponse(r, self)
class UsergridUser(object):
[16/50] [abbrv] usergrid git commit: reading token from SecurityUtils
rather than queryParams.
Posted by mr...@apache.org.
reading token from SecurityUtils rather than queryParams.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d393445f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d393445f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d393445f
Branch: refs/heads/master
Commit: d393445f19b214788a3b75ea74fb5e93265bedf4
Parents: 5c54d15
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Mon Jul 11 16:40:45 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Mon Jul 11 16:40:45 2016 -0700
----------------------------------------------------------------------
.../rest/management/ManagementResource.java | 23 +++++++++++++++++++-
.../rest/management/users/UserResource.java | 6 +++--
.../cassandra/ManagementServiceImpl.java | 4 +++-
3 files changed, 29 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d393445f/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 056303a..c94987a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -22,6 +22,7 @@ import org.apache.amber.oauth2.common.exception.OAuthProblemException;
import org.apache.amber.oauth2.common.message.OAuthResponse;
import org.apache.amber.oauth2.common.message.types.GrantType;
import org.apache.commons.lang.StringUtils;
+import org.apache.shiro.SecurityUtils;
import org.apache.shiro.codec.Base64;
import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.UserInfo;
@@ -34,6 +35,7 @@ import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.management.organizations.OrganizationsResource;
import org.apache.usergrid.rest.management.users.UsersResource;
import org.apache.usergrid.security.oauth.AccessInfo;
+import org.apache.usergrid.security.shiro.principals.PrincipalIdentifier;
import org.apache.usergrid.security.shiro.utils.SubjectUtils;
import org.apache.usergrid.security.sso.ExternalSSOProvider;
import org.apache.usergrid.security.sso.SSOProviderFactory;
@@ -99,6 +101,9 @@ public class ManagementResource extends AbstractContextResource {
MetricsFactory metricsFactory = null;
+ String access_token = null;
+
+
public ManagementResource() {
if (logger.isTraceEnabled()) {
logger.trace( "ManagementResource initialized" );
@@ -159,7 +164,6 @@ public class ManagementResource extends AbstractContextResource {
@QueryParam( "client_id" ) String client_id,
@QueryParam( "client_secret" ) String client_secret,
@QueryParam( "ttl" ) long ttl,
- @QueryParam( "access_token" ) String access_token,
@QueryParam( "callback" ) @DefaultValue( "" ) String callback )
throws Exception {
@@ -181,6 +185,12 @@ public class ManagementResource extends AbstractContextResource {
final boolean ssoEnabled = Boolean.parseBoolean(properties.getProperty(USERGRID_EXTERNAL_SSO_ENABLED));
long tokenTtl;
+ PrincipalIdentifier userPrincipal = (PrincipalIdentifier) SecurityUtils.getSubject().getPrincipal();
+ if ( userPrincipal != null && userPrincipal.getAccessTokenCredentials() != null ) {
+ this.access_token = userPrincipal.getAccessTokenCredentials().getToken();
+ }
+
+
if(ssoEnabled){
ExternalSSOProvider provider = ssoProviderFactory.getProvider();
@@ -384,6 +394,7 @@ public class ManagementResource extends AbstractContextResource {
@FormParam( "access_token" ) String access_token,
@FormParam( "callback" ) @DefaultValue( "" ) String callback )
throws Exception {
+
return getAccessTokenInternal( ui, authorization, grant_type, username, password, client_id, client_secret, ttl,
callback, false, true );
}
@@ -425,6 +436,16 @@ public class ManagementResource extends AbstractContextResource {
@QueryParam( "callback" ) @DefaultValue( "" ) String callback,
@HeaderParam( "Authorization" ) String authorization ) throws Exception {
+
+ if ( json == null ) {
+ String errorDescription = "invalid request, expected data in the request.";
+ OAuthResponse response =
+ OAuthResponse.errorResponse( SC_BAD_REQUEST ).setError( OAuthError.TokenResponse.INVALID_REQUEST )
+ .setErrorDescription( errorDescription ).buildJSONMessage();
+ return Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
+ .entity( wrapWithCallback( response.getBody(), callback ) ).build();
+ }
+
String grant_type = ( String ) json.get( "grant_type" );
String username = ( String ) json.get( "username" );
String password = ( String ) json.get( "password" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d393445f/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index 739ef28..f568463 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -64,7 +64,7 @@ public class UserResource extends AbstractContextResource {
String errorMsg;
- String token;
+ String token = null;
public UserResource() {
@@ -74,7 +74,9 @@ public class UserResource extends AbstractContextResource {
public UserResource init( UserInfo user ) {
this.user = user;
PrincipalIdentifier userPrincipal = (PrincipalIdentifier) SecurityUtils.getSubject().getPrincipal();
- this.token = userPrincipal.getAccessTokenCredentials().getToken();
+ if ( userPrincipal != null && userPrincipal.getAccessTokenCredentials() != null ) {
+ this.token = userPrincipal.getAccessTokenCredentials().getToken();
+ }
return this;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d393445f/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 9637dff..e812b75 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -1732,7 +1732,9 @@ public class ManagementServiceImpl implements ManagementService {
invalidateManagementAppAuthCache();
if ( email ) {
- sendAdminUserInvitedEmail( user, organization );
+ if(!tokens.isExternalSSOProviderEnabled()) {
+ sendAdminUserInvitedEmail(user, organization);
+ }
}
}
[26/50] [abbrv] usergrid git commit: Initial checkin for Python
Utilities and SDK
Posted by mr...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
new file mode 100644
index 0000000..37594d1
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
@@ -0,0 +1,119 @@
+import argparse
+import json
+import datetime
+import os
+import time
+import sys
+
+import boto
+from boto import sqs
+
+### This monitors an SQS queue and measures the delta message count between polling intervals to infer the amount of time
+### remaining to fully drain the queue
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+
+def total_seconds(td):
+ return (td.microseconds + (td.seconds + td.days * 24.0 * 3600) * 10.0 ** 6) / 10.0 ** 6
+
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+
+def get_time_remaining(count, rate):
+ if rate == 0:
+ return 'NaN'
+
+ seconds = count * 1.0 / rate
+
+ m, s = divmod(seconds, 60)
+ h, m = divmod(m, 60)
+
+ return "%d:%02d:%02d" % (h, m, s)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Loader - Queue Monitor')
+
+ parser.add_argument('-c', '--config',
+ help='The queue to load into',
+ type=str,
+ default='%s/.usergrid/queue_monitor.json' % os.getenv("HOME"))
+
+ parser.add_argument('-q', '--queue_name',
+ help='The queue name to send messages to. If not specified the filename is used',
+ default='entities',
+ type=str)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ print str(my_args)
+
+ return vars(my_args)
+
+
+def main():
+
+ args = parse_args()
+
+ queue_name = args.get('queue_name')
+
+ print 'queue_name=%s' % queue_name
+
+ start_time = datetime.datetime.utcnow()
+ first_start_time = start_time
+
+ print "first start: %s" % first_start_time
+
+ with open(args.get('config'), 'r') as f:
+ config = json.load(f)
+
+ sqs_config = config.get('sqs')
+ last_time = datetime.datetime.utcnow()
+
+ sqs_conn = boto.sqs.connect_to_region(**sqs_config)
+
+ queue = sqs_conn.get_queue(queue_name)
+
+ last_size = queue.count()
+ first_size = last_size
+
+ print 'Starting Size: %s' % last_size
+
+ sleep = 10
+ time.sleep(sleep)
+ rate_sum = 0
+ rate_count = 0
+
+ while True:
+ size = queue.count()
+ time_stop = datetime.datetime.utcnow()
+
+ time_delta = total_seconds(time_stop - last_time)
+ agg_time_delta = total_seconds(time_stop - first_start_time)
+ agg_size_delta = first_size - size
+ agg_messages_rate = 1.0 * agg_size_delta / agg_time_delta
+
+ size_delta = last_size - size
+ messages_rate = 1.0 * size_delta / time_delta
+ rate_sum += messages_rate
+ rate_count += 1
+
+ print '%s | %s | Size: %s | Processed: %s | Last: %s | Avg: %s | Count: %s | agg rate: %s | Remaining: %s' % (
+ datetime.datetime.utcnow(),
+ queue_name,
+ size, size_delta, round(messages_rate, 2),
+ round(rate_sum / rate_count, 2), rate_count,
+ round(agg_messages_rate, 2),
+ get_time_remaining(size, agg_messages_rate))
+
+ last_size = size
+ last_time = time_stop
+
+ time.sleep(sleep)
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
new file mode 100644
index 0000000..62755df
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
@@ -0,0 +1,87 @@
+import datetime
+import time
+
+import numpy
+import requests
+
+## This will call a URL over and over to check the latency of the call
+
+def total_milliseconds(td):
+ return (td.microseconds + td.seconds * 1000000) / 1000
+
+url_template = "{protocol}://{host}:{port}/{org}/{app}/{collection}?ql={ql}&client_id={client_id}&client_secret={client_secret}"
+
+environments = {
+
+ 'local': {
+ 'protocol': 'http',
+ 'host': 'localhost',
+ 'port': 8080,
+ 'org': 'myOrg',
+ 'app': 'myApp',
+ 'collection': 'myEntities',
+ 'ql': 'select *',
+ 'client_id': '<<client_id>>',
+ 'client_secret': '<<client_secret>>'
+ }
+}
+
+ENV = 'local'
+
+data = environments.get(ENV)
+if data is None:
+ print 'didn\'t find map entry for data'
+ exit(1)
+
+x = 0
+
+SLEEP = .5
+count_under_one = 0.0
+count_over = 0.0
+percent_under_one = 100.0
+total_time = 0
+
+print url_template.format(**data)
+
+response_times = []
+
+while True:
+ x += 1
+ target_url = url_template.format(**data)
+
+ r = requests.get(url=target_url)
+
+ response_time = total_milliseconds(r.elapsed)
+ total_time += response_time
+
+ # print '%s / %s' % (r.elapsed, total_milliseconds(r.elapsed))
+
+ the_date = datetime.datetime.utcnow()
+
+ if r.status_code != 200:
+ print '%s | %s: %s in %s | %s' % (the_date, x, r.status_code, response_time, r.text)
+ else:
+ response_times.append(response_time)
+
+ if response_time < 2000:
+ count_under_one += 1
+ elif response_time > 10000:
+ count_over += 1
+
+ percent_under_one = round(100 * (count_under_one / x), 2)
+ percent_over = round(100 * (count_over / x), 2)
+
+ # print '%s | %s: %s in %s | Count: %s | Avg: %s | under 2s: %s / %s%% | over 10s: %s / %s%%' % (
+ # the_date, x, r.status_code, response_time, len(r.json().get('entities')), (total_time / x), count_under_one,
+ # percent_under_one, count_over, percent_over)
+
+ print '%s | %s: %s in %s | Count: %s | Avg: %s | 99th: %s | 90th: %s | 50th: %s | 75th: %s | 25th: %s' % (
+ the_date, x, r.status_code, response_time, r.json().get('count'), (total_time / x),
+
+ numpy.percentile(response_times, 99),
+ numpy.percentile(response_times, 90),
+ numpy.percentile(response_times, 75),
+ numpy.percentile(response_times, 50),
+ numpy.percentile(response_times, 25))
+
+ time.sleep(SLEEP)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
new file mode 100644
index 0000000..9b1484e
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
@@ -0,0 +1,29 @@
+import json
+
+import requests
+
+### This will make the API calls to activate and confirm an array of users
+
+users = [
+ 'user1@example.com',
+ 'user2@example.com'
+]
+
+TOKEN = 'ABC123'
+URL = "http://localhost:8080/management/users/%s"
+
+s = requests.Session()
+s.headers.update({'authorization': 'Bearer %s' % TOKEN})
+
+for user in users:
+
+ r = s.put(URL % user, data=json.dumps({"activated": True}))
+ print 'Activated %s: %s' % (user, r.status_code)
+
+ if r.status_code != 200:
+ print r.text
+ continue
+
+ r = s.put(URL % user, data=json.dumps({"confirmed": True}))
+
+ print 'Confirmed %s: %s' % (user, r.status_code)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
new file mode 100644
index 0000000..d21183d
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
@@ -0,0 +1,49 @@
+import json
+import requests
+
+### This will create an array of org-level management users
+
+users = [
+ 'me@example.com'
+]
+
+for user in users:
+
+ post_body = {
+ "username": user,
+ "name": user,
+ "email": user,
+ "password": "test12345"
+ }
+
+ print json.dumps(post_body)
+
+ r = requests.post('http://localhost:8080/management/organizations/asdf/users',
+ headers={
+ 'Authorization': 'Bearer SADFSDF',
+ 'Content-Type': 'application/json'
+ },
+ data=json.dumps(post_body))
+
+ print r.status_code
+
+ print '%s: created (POST) [%s]: %s' % (user, r.status_code, r.text)
+
+ #
+ # r = requests.put('http://localhost:8080/management/users/%s' % user,
+ # headers={
+ # 'Authorization': 'Bearer YWMtFlVrhK8nEeW-AhmxdmpAVAAAAVIYTHxTNSUxpQyUWZQ2LsZxcXSdNtO_lWo',
+ # 'Content-Type': 'application/json'
+ # },
+ # data=json.dumps('{"confirmed": true}'))
+ #
+ # print '%s: confirmed: %s' % (user, r.status_code)
+ #
+ # r = requests.put('http://localhost:8080/management/users/%s' % user,
+ # headers={
+ # 'Authorization': 'Bearer YWMtFlVrhK8nEeW-AhmxdmpAVAAAAVIYTHxTNSUxpQyUWZQ2LsZxcXSdNtO_lWo',
+ # 'Content-Type': 'application/json'
+ # },
+ # data=json.dumps('{"activated": true}'))
+ #
+ # print '%s: activated: %s' % (user, r.status_code)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/groups/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/groups/__init__.py b/utils/usergrid-util-python/usergrid_tools/groups/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
new file mode 100644
index 0000000..268b7f8
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
@@ -0,0 +1,86 @@
+import json
+import traceback
+from multiprocessing import Pool
+
+import datetime
+import urllib3
+
+import requests
+
+group_name = 'precisely-10k'
+users = 10000
+username_template = 'precisely-10k-%s'
+
+url_data = {
+ "api_url": "https://usergrid-e2e-prod.e2e.apigee.net/appservices-2-1/",
+ "org": "tempgrey",
+ "app": "sandbox",
+ "client_id": "",
+ "client_secret": "",
+
+}
+
+url_data = {
+ "api_url": "http://baas-ug002sr.apigee.net/",
+ "org": "apigee-vfmplus",
+ "app": "sandbox",
+ "client_id": "",
+ "client_secret": "",
+
+}
+
+collection_url_template = "{api_url}/{org}/{app}/{collection}"
+add_user_url_template = "{api_url}/{org}/{app}/groups/{group_name}/users/{uuid}"
+
+
+def create_group(name):
+ url = collection_url_template.format(collection='groups', **url_data)
+ print url
+ r = requests.post(url, data=json.dumps({"path": name, "name": name}))
+
+ if r.status_code not in [200, 400]:
+ print r.text
+ exit()
+
+
+def create_user(username):
+ url = collection_url_template.format(collection='users', **url_data)
+ r = requests.post(url, data=json.dumps({"username": username}))
+
+ if r.status_code not in [200, 400]:
+ print r.text
+ exit()
+
+ print 'Created user %s' % username
+
+
+def map_user(username):
+ try:
+ url = add_user_url_template.format(group_name=group_name, uuid=username, **url_data)
+ r = requests.post(url, data=json.dumps({"username": username}))
+
+ if r.status_code != 200:
+ print r.text
+ exit()
+
+ print 'Mapped user %s' % username
+ except:
+ print traceback.format_exc()
+
+
+user_names = [username_template % i for i in xrange(0, users)]
+
+pool = Pool(64)
+
+start = datetime.datetime.utcnow()
+pool.map(create_user, user_names)
+
+create_group(group_name)
+
+pool.map(map_user, user_names)
+
+finish = datetime.datetime.utcnow()
+
+td = finish - start
+
+print td
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/indexing/README.md
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/README.md b/utils/usergrid-util-python/usergrid_tools/indexing/README.md
new file mode 100644
index 0000000..e938c28
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/README.md
@@ -0,0 +1,22 @@
+# Usergrid Indexing Latency Tester
+
+
+# Overview
+
+Indexing of data (to Elasticsearch) in Usergrid is done asynchronously, while persistence (to Cassandra) is done synchronously within the context of an API call. This means that you can immediately get your data back by UUID but if you use `GET /org/app/collection?ql=select * where field='value'` it is not instantly indexed. The typical delay is ~25ms.
+
+The purpose of this tool is to test the indexing latency within Usergrid.
+
+```
+$ usergrid_index_test -h
+
+usage: usergrid_index_test [-h] -o ORG -a APP --base_url BASE_URL
+
+Usergrid Indexing Latency Test
+
+optional arguments:
+ -h, --help show this help message and exit
+ -o ORG, --org ORG Name of the org to perform the test in
+ -a APP, --app APP Name of the app to perform the test in
+ --base_url BASE_URL The URL of the Usergrid Instance
+```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py b/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
new file mode 100644
index 0000000..6c910dd
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
@@ -0,0 +1,340 @@
+# -*- coding: utf-8 -*-
+import json
+import logging
+import traceback
+from multiprocessing import Pool
+import datetime
+import socket
+
+import argparse
+import requests
+import time
+from logging.handlers import RotatingFileHandler
+
+import sys
+
+entity_template = {
+ "id": "replaced",
+ "dataType": "entitlements",
+ "mockData": [
+ {"importDate": "2015-08-25T23:33:57.124Z", "rowsImported": 2},
+ {"role": "line-owner", "route": "/master", "element": "element1", "entitlementId": "entitlement4",
+ "property": "show"},
+ {"role": "line-owner", "route": "/master", "element": "element2", "entitlementId": "entitlement8",
+ "property": "hide"}
+ ],
+ "nullArray1": [None],
+ "nullArray2": [None, None],
+ "nullArray3": [None, None],
+ "nest1": {
+ "nest2": {
+ "nest3": [None, None, 'foo']
+ }
+ }
+}
+
+entity_template = {
+ "type": "customerstatuses",
+ "created": 1454769737888,
+ "modified": 1454781811473,
+ "address": {
+ "zip": "35873",
+ "city": "m�laga",
+ "street": "3430 calle de bravo murillo",
+ "state": "melilla"
+ },
+ "DOB": "787264244",
+ "email": "bego�a.caballero29@example.com",
+ "firstName": "Bego�a",
+ "lastName": "Caballero",
+ "lastSeenDateTime": 1447737158857,
+ "locationStatus": "Entrance",
+ "loyaltyAccountNumber": "1234",
+ "loyaltyLevel": "basic",
+ "phone": "966-450-469",
+ "profilePictureUrl": "http://api.randomuser.me/portraits/thumb/women/61.jpg",
+ "status": "Entrance",
+ "storeId": 12121
+}
+
+url_template = '{api_url}/{org}/{app}/{collection}'
+token_url_template = '{api_url}/{org}/{app}/token'
+
+config = {}
+
+session = requests.Session()
+
+logger = logging.getLogger('UsergridBatchIndexTest')
+
+
+def init_logging(stdout_enabled=True):
+ root_logger = logging.getLogger()
+ log_file_name = './usergrid_index_test.log'
+ log_formatter = logging.Formatter(fmt='%(asctime)s | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=2048576000,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+ root_logger.setLevel(logging.INFO)
+
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
+
+ if stdout_enabled:
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+ root_logger.addHandler(stdout_logger)
+
+
+def create_entity(work_item):
+ global config
+ try:
+ url = work_item[0]
+ entity = work_item[1]
+
+ # entity['name'] = datetime.datetime.now().strftime('name-%yx%mx%dx%Hx%Mx%S')
+
+ logger.info('creating entity [%s] at URL [%s]' % (entity.get('id'), url))
+
+ r = session.post(url, data=json.dumps(entity))
+
+ if r.status_code != 200:
+ logger.error('HTTP %s: %s' % (r.status_code, r.text))
+ print 'HTTP %s: %s' % (r.status_code, r.text)
+ return
+
+ entities = r.json().get('entities', [])
+ uuid = entities[0].get('uuid')
+
+ if r.status_code != 200:
+ logger.info('%s: %s' % (r.status_code, uuid))
+ else:
+ logger.info('Created entity UUID=[%s] at URL [%s]' % (uuid, url))
+
+ return uuid, entity
+
+ except Exception, e:
+ print traceback.format_exc(e)
+
+
+def test_multiple(number_of_entities):
+ global config
+
+ start = datetime.datetime.now()
+
+ logger.info('Creating %s entities w/ url=%s' % (number_of_entities, config['url']))
+ created_map = {}
+
+ work_items = []
+
+ for x in xrange(1, number_of_entities + 1):
+ entity = entity_template.copy()
+ entity['id'] = str(x)
+ work_items.append((config['url'], entity))
+
+ responses = processes.map(create_entity, work_items)
+
+ for res in responses:
+ if len(res) > 0:
+ created_map[res[0]] = res[1]
+
+ stop = datetime.datetime.now()
+
+ logger.info('Created [%s] entities in %s' % (number_of_entities, (stop - start)))
+
+ return created_map
+
+
+def wait_for_indexing(created_map, q_url, sleep_time=0.0):
+ logger.info('Waiting for indexing of [%s] entities...' % len(created_map))
+
+ count_missing = 100
+ start_time = datetime.datetime.now()
+
+ while count_missing > 0:
+
+ entity_map = {}
+ r = session.get(q_url)
+ res = r.json()
+ entities = res.get('entities', [])
+
+ now_time = datetime.datetime.now()
+ elapsed = now_time - start_time
+
+ logger.info('Found [%s] of [%s] ([%s] missing) after [%s] entities at url: %s' % (
+ len(entities), len(created_map), (len(created_map) - len(entities)), elapsed, q_url))
+
+ count_missing = 0
+
+ for entity in entities:
+ entity_map[entity.get('uuid')] = entity
+
+ for uuid, created_entity in created_map.iteritems():
+ if uuid not in entity_map:
+ count_missing += 1
+ logger.info('Missing uuid=[%s] Id=[%s] total missing=[%s]' % (
+ uuid, created_entity.get('id'), count_missing))
+
+ if count_missing > 0:
+ logger.info('Waiting for indexing, count_missing=[%s] Total time [%s] Sleeping for [%s]s' % (
+ elapsed, count_missing, sleep_time))
+
+ time.sleep(sleep_time)
+
+ stop_time = datetime.datetime.now()
+ logger.info('All entities found after %s' % (stop_time - start_time))
+
+
+def clear(clear_url):
+ logger.info('deleting.... ' + clear_url)
+
+ r = session.delete(clear_url)
+
+ if r.status_code != 200:
+ logger.info('error deleting url=' + clear_url)
+ logger.info(json.dumps(r.json()))
+
+ else:
+ res = r.json()
+ len_entities = len(res.get('entities', []))
+
+ if len_entities > 0:
+ clear(clear_url)
+
+
+def test_cleared(q_url):
+ r = session.get(q_url)
+
+ if r.status_code != 200:
+ logger.info(json.dumps(r.json()))
+ else:
+ res = r.json()
+
+ if len(res.get('entities', [])) != 0:
+ logger.info('DID NOT CLEAR')
+
+
+processes = Pool(32)
+
+
+def test_url(q_url, sleep_time=0.25):
+ test_var = False
+
+ while not test_var:
+ r = session.get(q_url)
+
+ if r.status_code == 200:
+
+ if len(r.json().get('entities')) >= 1:
+ test_var = True
+ else:
+ logger.info('non 200')
+
+ if test_var:
+ logger.info('Test of URL [%s] Passes')
+ else:
+ logger.info('Test of URL [%s] Passes')
+ time.sleep(sleep_time)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Indexing Latency Test')
+
+ parser.add_argument('-o', '--org',
+ help='Name of the org to perform the test in',
+ type=str,
+ required=True)
+
+ parser.add_argument('-a', '--app',
+ help='Name of the app to perform the test in',
+ type=str,
+ required=True)
+
+ parser.add_argument('--base_url',
+ help='The URL of the Usergrid Instance',
+ type=str,
+ required=True)
+
+ parser.add_argument('--client_id',
+ help='The Client ID to get a token, if needed',
+ type=str,
+ required=False)
+
+ parser.add_argument('--client_secret',
+ help='The Client Secret to get a token, if needed',
+ type=str,
+ required=False)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+def init():
+ global config
+
+ url_data = {
+ 'api_url': config.get('base_url'),
+ 'org': config.get('org'),
+ 'app': config.get('app'),
+ 'collection': '%s-%s' % (socket.gethostname(), datetime.datetime.now().strftime('index-test-%yx%mx%dx%Hx%Mx%S'))
+ }
+
+ config['url'] = url_template.format(**url_data)
+ config['token_url'] = token_url_template.format(**url_data)
+
+
+def main():
+ global config
+
+ # processes = Pool(32)
+
+ config = parse_args()
+
+ init()
+
+ init_logging()
+
+ if config.get('client_id') is not None and config.get('client_secret') is not None:
+ token_request = {
+ 'grant_type': 'client_credentials',
+ 'client_id': config.get('client_id'),
+ 'client_secret': config.get('client_secret')
+ }
+
+ r = session.post(config.get('token_url'), json.dumps(token_request))
+
+ if r.status_code == 200:
+ access_token = r.json().get('access_token')
+ session.headers.update({'Authorization': 'Bearer %s' % access_token})
+ else:
+ logger.critical('unable to get token: %s' % r.text)
+ exit(1)
+
+ try:
+ created_map = test_multiple(999)
+
+ q_url = config.get('url') + "?ql=select * where dataType='entitlements'&limit=1000"
+
+ wait_for_indexing(created_map=created_map,
+ q_url=q_url,
+ sleep_time=1)
+
+ delete_q_url = config.get('url') + "?ql=select * where dataType='entitlements'&limit=1000"
+
+ clear(clear_url=delete_q_url)
+
+ except KeyboardInterrupt:
+ processes.terminate()
+
+ processes.terminate()
+
+
+main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
new file mode 100644
index 0000000..6f193a6
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
@@ -0,0 +1,317 @@
+# -*- coding: utf-8 -*-
+import json
+import logging
+from multiprocessing import Pool
+import datetime
+
+import argparse
+import requests
+import time
+from logging.handlers import RotatingFileHandler
+
+import sys
+
+entity_template = {
+ "id": "replaced",
+ "dataType": "entitlements",
+ "mockData": [
+ {"importDate": "2015-08-25T23:33:57.124Z", "rowsImported": 2},
+ {"role": "line-owner", "route": "/master", "element": "element1", "entitlementId": "entitlement4",
+ "property": "show"},
+ {"role": "line-owner", "route": "/master", "element": "element2", "entitlementId": "entitlement8",
+ "property": "hide"}
+ ],
+ "nullArray1": [None],
+ "nullArray2": [None, None],
+ "nullArray3": [None, None],
+ "nest1": {
+ "nest2": {
+ "nest3": [None, None, 'foo']
+ }
+ }
+}
+
+entity_template = {
+ "type": "customerstatuses",
+ "name": "1234",
+ "created": 1454769737888,
+ "modified": 1454781811473,
+ "address": {
+ "zip": "35873",
+ "city": "m�laga",
+ "street": "3430 calle de bravo murillo",
+ "state": "melilla"
+ },
+ "DOB": "787264244",
+ "email": "bego�a.caballero29@example.com",
+ "firstName": "Bego�a",
+ "lastName": "Caballero",
+ "lastSeenDateTime": 1447737158857,
+ "locationStatus": "Entrance",
+ "loyaltyAccountNumber": "1234",
+ "loyaltyLevel": "basic",
+ "phone": "966-450-469",
+ "profilePictureUrl": "http://api.randomuser.me/portraits/thumb/women/61.jpg",
+ "status": "Entrance",
+ "storeId": 12121
+}
+
+collection_url_template = '{api_url}/{org}/{app}/{collection}'
+query_url_template = '{api_url}/{org}/{app}/{collection}?ql=select * where tag=\'{tag}\''
+entity_url_template = '{api_url}/{org}/{app}/{collection}/{entity_id}'
+token_url_template = '{api_url}/{org}/{app}/token'
+
+config = {}
+
+session = requests.Session()
+
+logger = logging.getLogger('UsergridEntityIndexTest')
+
+
+def init_logging(stdout_enabled=True):
+ root_logger = logging.getLogger()
+ log_file_name = './usergrid_index_test.log'
+ log_formatter = logging.Formatter(fmt='%(asctime)s | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=2048576000,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+ root_logger.setLevel(logging.INFO)
+
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
+
+ if stdout_enabled:
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+ root_logger.addHandler(stdout_logger)
+
+
+def test_multiple(number_of_entities, processes):
+ global config
+
+ start = datetime.datetime.now()
+
+ logger.info('Creating %s entities w/ url=%s' % (number_of_entities, config['url']))
+ created_map = {}
+
+ work_items = []
+
+ for x in xrange(1, number_of_entities + 1):
+ entity = entity_template.copy()
+ entity['id'] = str(x)
+ work_items.append((config['url'], entity))
+
+ responses = processes.map(create_entity, work_items)
+
+ for res in responses:
+ if len(res) > 0:
+ created_map[res[0]] = res[1]
+
+ stop = datetime.datetime.now()
+
+ logger.info('Created [%s] entities in %s' % (number_of_entities, (stop - start)))
+
+ return created_map
+
+
+def clear(clear_url):
+ logger.info('deleting.... ' + clear_url)
+
+ r = session.delete(clear_url)
+
+ if r.status_code != 200:
+ logger.info('error deleting url=' + clear_url)
+ logger.info(json.dumps(r.json()))
+
+ else:
+ res = r.json()
+ len_entities = len(res.get('entities', []))
+
+ if len_entities > 0:
+ clear(clear_url)
+
+
+def test_cleared(q_url):
+ r = session.get(q_url)
+
+ if r.status_code != 200:
+ logger.info(json.dumps(r.json()))
+ else:
+ res = r.json()
+
+ if len(res.get('entities', [])) != 0:
+ logger.info('DID NOT CLEAR')
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Indexing Latency Test')
+
+ parser.add_argument('-o', '--org',
+ help='Name of the org to perform the test in',
+ type=str,
+ required=True)
+
+ parser.add_argument('-a', '--app',
+ help='Name of the app to perform the test in',
+ type=str,
+ required=True)
+
+ parser.add_argument('--base_url',
+ help='The URL of the Usergrid Instance',
+ type=str,
+ required=True)
+
+ parser.add_argument('--client_id',
+ help='The Client ID to get a token, if needed',
+ type=str,
+ required=False)
+
+ parser.add_argument('--client_secret',
+ help='The Client Secret to get a token, if needed',
+ type=str,
+ required=False)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+def init():
+ global config
+
+ url_data = {
+ 'api_url': config.get('base_url'),
+ 'org': config.get('org'),
+ 'app': config.get('app'),
+ 'collection': datetime.datetime.now().strftime('index-test-%yx%mx%dx%Hx%Mx%S')
+ }
+
+ config['url_data'] = url_data
+ config['token_url'] = token_url_template.format(**url_data)
+
+
+def create_entity(name, tag):
+ create_me = entity_template.copy()
+ start_tag = datetime.datetime.now().strftime('tag-%yx%mx%dx%Hx%Mx%S')
+ create_me['tag'] = start_tag
+
+ data = config.get('url_data')
+ url = collection_url_template.format(**data)
+
+ r = session.post(url, data=json.dumps(create_me))
+
+ if r.status_code != 200:
+ logger.critical('unable to create entity: %s' % r.text)
+ return None
+ else:
+ return r.json().get('entities')[0]
+
+
+def update_entity(entity_id, tag):
+ data = {'tag': tag}
+ url = entity_url_template.format(entity_id=entity_id, **config.get('url_data'))
+ r = session.put(url, data=json.dumps(data))
+
+ if r.status_code != 200:
+ logger.critical('unable to update entity!')
+ return False
+ else:
+ return True
+
+
+def wait_for_index(entity_id, tag, wait_time=.25):
+ start = datetime.datetime.now()
+
+ url = query_url_template.format(tag=tag, **config.get('url_data'))
+
+ logger.info('GET %s' % url)
+
+ entities = []
+ elapsed = 0
+
+ while len(entities) <= 0:
+ r = session.get(url)
+
+ if r.status_code != 200:
+ logger.critical('Unable to query, url=[%s]: %s' % (url, r.text))
+ return False
+ else:
+ res = r.json()
+ entities = res.get('entities')
+ last_time = datetime.datetime.now()
+ elapsed = last_time - start
+ logger.info(
+ 'Tag [%s] not applied to [%s] after [%s]. Waiting [%s]...' % (tag, entity_id, elapsed, wait_time))
+ time.sleep(wait_time)
+
+ logger.info('++Tag applied after [%s]!' % elapsed)
+
+
+def test_entity_update():
+ start_tag = datetime.datetime.now().strftime('tag-%yx%mx%dx%Hx%Mx%S')
+ name = datetime.datetime.now().strftime('name-%yx%mx%dx%Hx%Mx%S')
+ entity = create_entity(name, start_tag)
+
+ if entity is None:
+ logger.critical('Entity not created, cannot continue')
+ return
+
+ uuid = entity.get('uuid')
+
+ for x in xrange(0, 10):
+ tag = datetime.datetime.now().strftime('tag-%yx%mx%dx%Hx%Mx%S')
+ logger.info('Testing tag [%s] on entity [%s]' % (tag, name))
+ updated = update_entity(name, tag)
+ if updated: wait_for_index(name, tag)
+
+ for x in xrange(0, 10):
+ tag = datetime.datetime.now().strftime('tag-%yx%mx%dx%Hx%Mx%S')
+ logger.info('Testing tag [%s] on entity [%s]' % (tag, uuid))
+ updated = update_entity(uuid, tag)
+ if updated: wait_for_index(uuid, tag)
+
+
+def main():
+ global config
+
+ processes = Pool(32)
+
+ config = parse_args()
+
+ init()
+
+ init_logging()
+
+ if config.get('client_id') is not None and config.get('client_secret') is not None:
+ token_request = {
+ 'grant_type': 'client_credentials',
+ 'client_id': config.get('client_id'),
+ 'client_secret': config.get('client_secret')
+ }
+
+ r = session.post(config.get('token_url'), json.dumps(token_request))
+
+ if r.status_code == 200:
+ access_token = r.json().get('access_token')
+ session.headers.update({'Authorization': 'Bearer %s' % access_token})
+ else:
+ logger.critical('unable to get token: %s' % r.text)
+ exit(1)
+
+ try:
+ test_entity_update()
+
+ except KeyboardInterrupt:
+ pass
+ processes.terminate()
+
+
+main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/iterators/README.md
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/README.md b/utils/usergrid-util-python/usergrid_tools/iterators/README.md
new file mode 100644
index 0000000..cf61d4c
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/README.md
@@ -0,0 +1,8 @@
+simple_iterator
+---------------
+Basis for iterating a collection or all pages of a query and doing something with the data, such as counting or modifying
+
+
+usergrid_cross_region_iterator
+---------------
+used to iterate data and check that it exists in another region
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/iterators/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/__init__.py b/utils/usergrid-util-python/usergrid_tools/iterators/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
new file mode 100644
index 0000000..60cc4a0
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
@@ -0,0 +1,79 @@
+import logging
+import sys
+import uuid
+from logging.handlers import RotatingFileHandler
+
+import datetime
+from usergrid import UsergridQueryIterator
+
+execution_id = str(uuid.uuid4())
+
+
+def init_logging(stdout_enabled=True):
+ root_logger = logging.getLogger()
+ root_logger.setLevel(logging.INFO)
+
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.ERROR)
+ logging.getLogger('boto').setLevel(logging.ERROR)
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+
+ log_formatter = logging.Formatter(
+ fmt='%(asctime)s | ' + execution_id + ' | %(name)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.CRITICAL)
+ root_logger.addHandler(stdout_logger)
+
+ if stdout_enabled:
+ stdout_logger.setLevel(logging.INFO)
+
+ # base log file
+
+ log_dir = './'
+ log_file_name = '%s/usergrid_iterator.log' % log_dir
+
+ # ConcurrentLogHandler
+ rotating_file = RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=404857600,
+ backupCount=0)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+
+
+def main():
+ init_logging()
+
+ logger = logging.getLogger('SimpleIterator')
+
+ if len(sys.argv) <= 1:
+ logger.critical('usage: usergrid_iterator {url}')
+ exit(1)
+
+ url = sys.argv[1]
+ logger.info('Beginning to iterate URL: %s' % url)
+
+ q = UsergridQueryIterator(url)
+
+ counter = 0
+
+ start = datetime.datetime.utcnow()
+ try:
+ for e in q:
+ counter += 1
+ logger.info('Entity # [%s]: name=[%s] uuid=[%s] created=[%s] modified=[%s]' % (counter, e.get('name'), e.get('uuid'), e.get('created'), e.get('modified')))
+
+ except KeyboardInterrupt:
+ logger.critical('KEYBOARD INTERRUPT')
+ pass
+
+ finish = datetime.datetime.utcnow()
+
+ logger.info('final entity count is [%s] in [%s] for query [%s]' % (counter, (finish-start), url))
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
new file mode 100644
index 0000000..35933a2
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
@@ -0,0 +1,409 @@
+from usergrid import UsergridQuery
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+from Queue import Empty
+import argparse
+import json
+import time
+import logging
+import sys
+from multiprocessing import Process, JoinableQueue
+import datetime
+import requests
+import traceback
+from logging.handlers import RotatingFileHandler
+import urllib3
+import urllib3.contrib.pyopenssl
+
+urllib3.disable_warnings()
+urllib3.contrib.pyopenssl.inject_into_urllib3()
+
+
+# This was used to force a sync of C* across the regions. The idea is to query entities from
+# a region where they exist using QL. Then, iterate over the results and do a GET by UUID
+# in the region where the entities are 'missing'.
+#
+# In order for this to be successful the readcl in the "GET by UUID" region or target region
+# must be set to 'ALL' - this will force a repair across the cluster.
+#
+# It is recommended to have the target tomcat out of the ELB for a customer. Ideally,
+# you should spin up another Tomcat, leaving 2+ in the ELB for a given customer.
+
+
+logger = logging.getLogger('UsergridCrossRegionRepair')
+
+token_url_template = "{api_url}/management/token"
+org_management_url_template = "{api_url}/management/organizations/{org}/applications?access_token={access_token}"
+org_url_template = "{api_url}/{org}?access_token={access_token}"
+app_url_template = "{api_url}/{org}/{app}?access_token={access_token}"
+collection_url_template = "{api_url}/{org}/{app}/{collection}?access_token={access_token}"
+collection_query_url_template = "{api_url}/{org}/{app}/{collection}?ql={ql}&access_token={access_token}&limit={limit}"
+get_entity_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}?access_token={access_token}&connections=none"
+put_entity_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}?access_token={access_token}"
+
+# config can be loaded from a file
+config = {}
+
+# config = {
+# "regions": {
+# "us_west": {
+# "api_url": "http://rut040wo:8080"
+# },
+# "us_east": {
+# "api_url": "http://rut154ea:8080"
+# },
+# "eu_west": {
+# "api_url": "http://localhost:8080"
+# }
+# },
+# "management_region_id": "us_west",
+# "query_region_id": "us_west",
+# "get_region_ids": [
+# "us_east"
+# ]
+# }
+
+session_map = {}
+
+
+def init_logging(stdout_enabled=True):
+ root_logger = logging.getLogger()
+ log_file_name = './cross-region-repair.log'
+ log_formatter = logging.Formatter(fmt='%(asctime)s | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=204857600,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+ root_logger.setLevel(logging.INFO)
+
+ logging.getLogger('boto').setLevel(logging.ERROR)
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
+
+ if stdout_enabled:
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+ root_logger.addHandler(stdout_logger)
+
+
+class Worker(Process):
+ def __init__(self, queue, handler_function):
+ super(Worker, self).__init__()
+ logger.warning('Creating worker!')
+ self.queue = queue
+ self.handler_function = handler_function
+
+ def run(self):
+
+ logger.info('starting run()...')
+ keep_going = True
+
+ count_processed = 0
+ count_error = 0
+
+ while keep_going:
+ empty_count = 0
+
+ try:
+ org, app, collection, entity = self.queue.get(timeout=600)
+ logger.debug('Task: org=[%s] app=[%s] collection=[%s] entity=[%s]' % (org, app, collection, entity))
+
+ if self.handler_function is not None:
+ processed = self.handler_function(org=org,
+ app=app,
+ collection=collection,
+ entity=entity,
+ counter=count_processed)
+
+ if processed:
+ count_processed += 1
+ logger.info('Processed count=[%s] SUCCESS uuid/name = %s / %s' % (
+ count_processed, entity.get('uuid'), entity.get('name')))
+ else:
+ count_error += 1
+ logger.error('Processed count=[%s] ERROR uuid/name = %s / %s' % (
+ count_error, entity.get('uuid'), entity.get('name')))
+
+ self.queue.task_done()
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except Empty:
+ logger.warning('EMPTY!')
+ empty_count += 1
+ if empty_count > 30:
+ keep_going = False
+
+ logger.warning('WORKER DONE!')
+
+
+def wait_for(threads, sleep_time=3000):
+ count_alive = 1
+
+ while count_alive > 0:
+ count_alive = 0
+
+ for t in threads:
+
+ if t.is_alive():
+ count_alive += 1
+
+ if count_alive > 0:
+ logger.warning('Waiting for [%s] processes to finish' % count_alive)
+ time.sleep(sleep_time)
+
+
+def parse_args():
+ DEFAULT_WORKERS = 16
+ DEFAULT_TOKEN_TTL = 25200000
+
+ parser = argparse.ArgumentParser(description='Usergrid Cross-Region Repair Script')
+
+ parser.add_argument('-o', '--org',
+ help='The org to iterate',
+ type=str,
+ required=True)
+
+ parser.add_argument('-a', '--app',
+ help='The org to iterate',
+ action='append',
+ default=[])
+
+ parser.add_argument('-c', '--collection',
+ help='The org to iterate',
+ action='append',
+ default=[])
+
+ parser.add_argument('-p', '--password',
+ help='The Password for the token request',
+ type=str,
+ required=True)
+
+ parser.add_argument('-u', '--username',
+ help='The Username for the token request',
+ type=str,
+ required=True)
+
+ parser.add_argument('-w', '--workers',
+ help='The Password for the token request',
+ type=int,
+ default=DEFAULT_WORKERS)
+
+ parser.add_argument('--ttl',
+ help='The TTL for the token request',
+ type=int,
+ default=DEFAULT_TOKEN_TTL)
+
+ parser.add_argument('-l', '--limit',
+ help='The global limit for QL requests',
+ type=int,
+ default=DEFAULT_WORKERS * 3)
+
+ parser.add_argument('-f', '--config',
+ help='The file from which to load the configuration',
+ type=str)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+def get_by_UUID(org, app, collection, entity, counter, attempts=0):
+ response = False
+
+ if attempts >= 10:
+ return False
+
+ for region_id in config.get('get_region_ids', []):
+ url_data = config.get('regions', {}).get(region_id)
+
+ url = get_entity_url_template.format(collection=collection,
+ app=app,
+ uuid=entity.get('uuid'),
+ org=org,
+ access_token=config['access_token'],
+ **url_data)
+
+ logger.info('GET [%s]: %s' % ('...', url))
+
+ session = session_map[region_id]
+
+ while not response:
+
+ try:
+ r = session.get(url)
+
+ if r.status_code != 200:
+ logger.error('GET [%s] (%s): %s' % (r.status_code, r.elapsed, url))
+ logger.warning('Sleeping for 5 on connection retry...')
+
+ return get_by_UUID(org, app, collection, entity, counter, attempts=attempts + 1)
+
+ else:
+ logger.info('GET [%s] (%s): %s' % (r.status_code, r.elapsed, url))
+ response = True
+
+ if counter % 10 == 0:
+ logger.info('COUNTER=[%s] time=[%s] GET [%s]: %s' % (counter,
+ r.elapsed,
+ r.status_code,
+ url))
+ except:
+ logger.error(traceback.format_exc())
+ logger.error('EXCEPTION on GET [...] (...): %s' % url)
+ response = False
+ logger.warning('Sleeping for 5 on connection retry...')
+ time.sleep(5)
+
+ return response
+
+
+def init(args):
+ global config
+
+ if args.get('config') is not None:
+ config_filename = args.get('config')
+
+ logger.warning('Using config file: %s' % config_filename)
+
+ try:
+ with open(config_filename, 'r') as f:
+ parsed_config = json.load(f)
+ logger.warning('Updating config with: %s' % parsed_config)
+ config.update(parsed_config)
+ except:
+ print traceback.format_exc()
+
+ for region_id, region_data in config.get('regions', {}).iteritems():
+ session_map[region_id] = requests.Session()
+
+
+def main():
+ global config
+
+ args = parse_args()
+ init(args)
+
+ management_region_id = config.get('management_region_id', '')
+ management_region = config.get('regions', {}).get(management_region_id)
+
+ query_region_id = config.get('query_region_id', '')
+ query_region = config.get('regions', {}).get(query_region_id)
+
+ start = datetime.datetime.now()
+
+ queue = JoinableQueue()
+
+ logger.warning('Starting workers...')
+ init_logging()
+
+ token_request = {
+ 'grant_type': 'password',
+ 'username': args.get('username'),
+ 'ttl': args.get('ttl')
+ }
+
+ url = token_url_template.format(**management_region)
+
+ logger.info('getting token with url=[%s] data=[%s]' % (url, token_request))
+
+ token_request['password'] = args.get('password')
+
+ r = requests.post(url, data=json.dumps(token_request))
+
+ if r.status_code != 200:
+ logger.critical('did not get access token! response: %s' % r.json())
+ exit(-1)
+
+ logger.info(r.json())
+
+ config['access_token'] = r.json().get('access_token')
+
+ org_mgmt_url = org_management_url_template.format(org=args.get('org'),
+ access_token=config['access_token'],
+ **management_region)
+ logger.info(org_mgmt_url)
+
+ session = session_map[management_region_id]
+
+ r = session.get(org_mgmt_url)
+ logger.info(r.json())
+ logger.info('starting [%s] workers...' % args.get('workers'))
+ workers = [Worker(queue, get_by_UUID) for x in xrange(args.get('workers'))]
+ [w.start() for w in workers]
+
+ try:
+ org_app_data = r.json().get('data')
+
+ logger.info(org_app_data)
+
+ apps_to_process = config.get('app', [])
+ collections_to_process = config.get('collection', [])
+
+ for org_app, app_uuid in org_app_data.iteritems():
+ parts = org_app.split('/')
+ app = parts[1]
+
+ if len(apps_to_process) > 0 and app not in apps_to_process:
+ logger.info('Skipping app/uuid: %s/%s' % (org_app, app_uuid))
+ continue
+
+ logger.info('app UUID: %s' % app_uuid)
+
+ url = app_url_template.format(app=app,
+ org=args.get('org'),
+ access_token=config['access_token'],
+ **management_region)
+
+ logger.info('GET [...]: %s' % url)
+ session = session_map[management_region_id]
+ r = session.get(url)
+
+ for collection_name in r.json().get('entities', [{}])[0].get('metadata', {}).get('collections', {}):
+
+ if collection_name in ['events']:
+ continue
+
+ elif len(collections_to_process) > 0 and collection_name not in collections_to_process:
+ logger.info('skipping collection=%s' % collection_name)
+ continue
+
+ logger.info('processing collection=%s' % collection_name)
+
+ url = collection_query_url_template.format(ql='select * order by created asc',
+ collection=collection_name,
+ org=args['org'],
+ app=app,
+ limit=args['limit'],
+ access_token=config['access_token'],
+ **query_region)
+
+ q = UsergridQuery(url)
+ counter = 0
+
+ for x, e in enumerate(q):
+ counter += 1
+ queue.put((args['org'], app, collection_name, e))
+
+ logger.info('collection=%s, count=%s' % (collection_name, counter))
+
+ except KeyboardInterrupt:
+ [w.terminate() for w in workers]
+
+ logger.warning('Waiting for workers to finish...')
+ wait_for(workers)
+
+ finish = datetime.datetime.now()
+ logger.warning('Done! Took: %s ' % (finish - start))
+
+
+main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/library_check.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/library_check.py b/utils/usergrid-util-python/usergrid_tools/library_check.py
new file mode 100644
index 0000000..f326987
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/library_check.py
@@ -0,0 +1,23 @@
+import traceback
+
+url_data = {
+ "api_url": "https://usergrid-e2e-prod.e2e.apigee.net/appservices-2-1/",
+ "org": "",
+ "app": "",
+ "client_id": "",
+ "client_secret": "",
+
+}
+
+collection_url_template = "{api_url}/{org}/{app}/{collection}"
+
+try:
+ from usergrid import UsergridQueryIterator
+
+ q = UsergridQueryIterator('')
+
+ print 'Check OK'
+
+except Exception, e:
+ print traceback.format_exc(e)
+ print 'Check Failed'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/migration/README.md
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/README.md b/utils/usergrid-util-python/usergrid_tools/migration/README.md
new file mode 100644
index 0000000..921b0a7
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/migration/README.md
@@ -0,0 +1,234 @@
+# Usergrid Data Migrator
+
+## Prerequisites
+* Python 2 (not python 3)
+
+* Install the Usergrid Python SDK: https://github.com/jwest-apigee/usergrid-python
+
+With Pip (requires python-pip to be installed): `pip install usergrid`
+
+* Install Usergrid Tools
+
+With Pip (requires python-pip to be installed): `pip install usergrid-tools`
+
+
+## Overview
+The purpose of this document is to provide an overview of the Python Script provided in the same directory which allows you to migrate data, connections and users from one Usergrid platform / org / app to another. This can be used in the upgrade process from Usergrid 1.0 to 2.x since there is no upgrade path.
+
+This script functions by taking source and target endpoint configurations (with credentials) and a set of command-line parameters to read data from one Usergrid instance and write to another. It is written in Python and requires Python 2.7.6+.
+
+There are multiple processes at work in the migration to speed the process up. There is a main thread which reads entities from the API and then publishes the entities with metadata into a Python Queue which has multiple worker processes listening for work. The number of worker threads is configurable by command line parameters.
+
+
+# Process to Migrate Data and Graph (Connections)
+Usergrid is a Graph database and allows for connections between entities. In order for a connection to be made, both the source entity and the target entity must exist. Therefore, in order to migrate connections it is adviseable to first migrate all the data and then all the connections associated with that data.
+
+# Concepts
+As with any migration process there is a source and a target. The source and target have the following parameters:
+
+* API URL: The HTTP[S] URL where the platform can be reached
+* Org: You must specify one org at a time to migrate using this script
+* App: You can optinally specify one or more applications to migrate. If you specify zero applications then all applications will be migrated
+* Collection: You can optionally specify one or more collections to migrate. If you specify zero collections then all applications will be migrated
+* QL: You can specify a Query Language predicate to be used. If none is specified, 'select *' will be used which will migrate all data within a given collection
+* Graph: Graph implies traversal of graph edges which necessarily must exist. This is an alternative to using query which uses the indexing.
+
+# Graph Loops
+
+When iterating a graph it is possible to get stuck in a loop. For example:
+
+```
+A --follows--> B
+B --likes--> C
+C --loves--> A
+```
+
+There are two options to prevent getting stuck in a loop:
+* `graph_depth` option - this will limit the graph depth which will be traversed from a given entity.
+* And/Or Marking nodes and edges as 'visited'. This requires a place to store this state. See Using Redis in the next section
+
+# Using Redis
+
+Redis can be used for the following:
+
+If using Redis, version 2.8+ is needed because TTL is used with the 'ex' parameter.
+
+* Keeping track of the modified date for each entity. When running the script subsequent times after this, entiites which were not modified will not be copied.
+* Keeping track of visited nodes for migrating a graph. This is done with a TTL such that a job can be resumed, but since there is no modified date on an edge you cannot know if there are new edges or not. Therefore, when the TTL expires the nodes will be visited again
+* Keeping track of the URLs for the connections which are created between entities. This has no TTL. Subsequent runs will not create connections which are found in Redis which have already been created.
+
+
+# Mapping
+Using this script it is not necessary to keep the same application name, org name and/or collection name as the source at the target. For example, you could migrate from /myOrg/myApp/myCollection to /org123/app456/collections789.
+
+
+# Configuration Files
+Example source/target configuration files:
+
+```
+{
+ "endpoint": {
+ "api_url": "https://api.usergrid.com"
+ },
+ "credentials": {
+ "myOrg1": {
+ "client_id": "YXA6lpg9sEaaEeONxG0g3Uz44Q",
+ "client_secret": "ZdF66u2h3Hc7csOcsEtgewmxalB1Ygg"
+ },
+ "myOrg2": {
+ "client_id": "ZXf63p239sDaaEeONSG0g3Uz44Z",
+ "client_secret": "ZdF66u2h3Hc7csOcsEtgewmxajsadfj32"
+ }
+ }
+}
+```
+* api_url: the API URL to access/write data
+* Credentials:
+ * For each org, with the org name (case-sensetive) as the key:
+ * client_id - the org-level Client ID. This can be retrieved from the BaaS/Usergrid Portal.
+ * client_secret - the org-level Client Secret. This can be retrieved from the BaaS/Usergrid Portal.
+
+# Command Line Parameters
+
+```
+Usergrid Org/App Data Migrator
+
+optional arguments:
+ -h, --help show this help message and exit
+ --log_dir LOG_DIR path to the place where logs will be written
+ --log_level LOG_LEVEL
+ log level - DEBUG, INFO, WARN, ERROR, CRITICAL
+ -o ORG, --org ORG Name of the org to migrate
+ -a APP, --app APP Name of one or more apps to include, specify none to
+ include all apps
+ -e INCLUDE_EDGE, --include_edge INCLUDE_EDGE
+ Name of one or more edges/connection types to INCLUDE,
+ specify none to include all edges
+ --exclude_edge EXCLUDE_EDGE
+ Name of one or more edges/connection types to EXCLUDE,
+ specify none to include all edges
+ --exclude_collection EXCLUDE_COLLECTION
+ Name of one or more collections to EXCLUDE, specify
+ none to include all collections
+ -c COLLECTION, --collection COLLECTION
+ Name of one or more collections to include, specify
+ none to include all collections
+ --use_name_for_collection USE_NAME_FOR_COLLECTION
+ Name of one or more collections to use [name] instead
+ of [uuid] for creating entities and edges
+ -m {data,none,reput,credentials,graph}, --migrate {data,none,reput,credentials,graph}
+ Specifies what to migrate: data, connections,
+ credentials, audit or none (just iterate the
+ apps/collections)
+ -s SOURCE_CONFIG, --source_config SOURCE_CONFIG
+ The path to the source endpoint/org configuration file
+ -d TARGET_CONFIG, --target_config TARGET_CONFIG
+ The path to the target endpoint/org configuration file
+ --limit LIMIT The number of entities to return per query request
+ -w ENTITY_WORKERS, --entity_workers ENTITY_WORKERS
+ The number of worker processes to do the migration
+ --visit_cache_ttl VISIT_CACHE_TTL
+ The TTL of the cache of visiting nodes in the graph
+ for connections
+ --error_retry_sleep ERROR_RETRY_SLEEP
+ The number of seconds to wait between retrieving after
+ an error
+ --page_sleep_time PAGE_SLEEP_TIME
+ The number of seconds to wait between retrieving pages
+ from the UsergridQueryIterator
+ --entity_sleep_time ENTITY_SLEEP_TIME
+ The number of seconds to wait between retrieving pages
+ from the UsergridQueryIterator
+ --collection_workers COLLECTION_WORKERS
+ The number of worker processes to do the migration
+ --queue_size_max QUEUE_SIZE_MAX
+ The max size of entities to allow in the queue
+ --graph_depth GRAPH_DEPTH
+ The graph depth to traverse to copy
+ --queue_watermark_high QUEUE_WATERMARK_HIGH
+ The point at which publishing to the queue will PAUSE
+ until it is at or below low watermark
+ --min_modified MIN_MODIFIED
+ Break when encountering a modified date before this,
+ per collection
+ --max_modified MAX_MODIFIED
+ Break when encountering a modified date after this,
+ per collection
+ --queue_watermark_low QUEUE_WATERMARK_LOW
+ The point at which publishing to the queue will RESUME
+ after it has reached the high watermark
+ --ql QL The QL to use in the filter for reading data from
+ collections
+ --skip_cache_read Skip reading the cache (modified timestamps and graph
+ edges)
+ --skip_cache_write Skip updating the cache with modified timestamps of
+ entities and graph edges
+ --create_apps Create apps at the target if they do not exist
+ --nohup specifies not to use stdout for logging
+ --graph Use GRAPH instead of Query
+ --su_username SU_USERNAME
+ Superuser username
+ --su_password SU_PASSWORD
+ Superuser Password
+ --inbound_connections
+ Name of the org to migrate
+ --map_app MAP_APP Multiple allowed: A colon-separated string such as
+ 'apples:oranges' which indicates to put data from the
+ app named 'apples' from the source endpoint into app
+ named 'oranges' in the target endpoint
+ --map_collection MAP_COLLECTION
+ One or more colon-separated string such as 'cats:dogs'
+ which indicates to put data from collections named
+ 'cats' from the source endpoint into a collection
+ named 'dogs' in the target endpoint, applicable
+ globally to all apps
+ --map_org MAP_ORG One or more colon-separated strings such as 'red:blue'
+ which indicates to put data from org named 'red' from
+ the source endpoint into a collection named 'blue' in
+ the target endpoint
+```
+
+## Example Command Line
+
+Use the following command to migrate DATA AND GRAPH (no graph edges or connections between entities). If there are no graph edges (connections) then using `-m graph` is not necessary. This will copy all data from all apps in the org 'myorg', creating apps in the target org if they do not already exist. Note that --create_apps will be required if the Apps in the target org have not been created.
+
+```
+$ usergrid_data_migrator -o myorg -m graph -w 4 -s mySourceConfig.json -d myTargetConfiguration.json --create_apps
+```
+
+Use the following command to migrate DATA ONLY (no graph edges or connections between entities). This will copy all data from all apps in the org 'myorg', creating apps in the target org if they do not already exist. Note that --create_apps will be required if the Apps in the target org have not been created.
+
+```
+$ usergrid_data_migrator -o myorg -m data -w 4 -s mySourceConfig.json -d myTargetConfiguration.json --create_apps
+```
+
+Use the following command to migrate CREDENTIALS for Application-level Users. Note that `usergrid.sysadmin.login.allowed=true` must be set in the `usergrid-deployment.properties` file on the source and target Tomcat nodes.
+
+```
+$ usergrid_data_migrator -o myorg -m credentails -w 4 -s mySourceConfig.json -d myTargetConfiguration.json --create_apps --su_username foo --su_password bar
+```
+
+This command:
+
+```
+$ usergrid_data_migrator -o myorg -a app1 -a app2 -m data -w 4 --map_app app1:appplication_1 --map_app app2:application_2 --map_collection pets:animals --map_org myorg:my_new_org -s mySourceConfig.json -d myTargetConfiguration.json
+```
+will do the following:
+
+* migrate Apps named 'app1' and 'app2' in org named 'myorg' from the API endpoint defined in 'mySourceConfig.json' to the API endpoint defined in 'myTargetConfiguration.json'
+* In the process:
+** data from 'myorg' will ge migrated to the org named 'my_new_org'
+** data from 'app1' will be migrated to the app named 'application_1'
+** data from 'app2' will be migrated to the app named 'application_2'
+** all collections named 'pets' will be overridden at the destination to 'animals'
+
+
+# FAQ
+
+### Does the process keep the same UUIDs?
+
+* Yes - with this script the same UUIDs can be kept from the source into the destination. An exception is if you specify going from one collection to another under the same Org hierarchy.
+
+### Does the process keep the ordering of connections by time?
+
+* Yes ordering of connections is maintained in the process.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/migration/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/__init__.py b/utils/usergrid-util-python/usergrid_tools/migration/__init__.py
new file mode 100644
index 0000000..f09d5b5
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/migration/__init__.py
@@ -0,0 +1,2 @@
+import usergrid_data_migrator
+import usergrid_data_exporter
\ No newline at end of file
[34/50] [abbrv] usergrid git commit: added apache header,
cleaned up some things.
Posted by mr...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py b/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
index 9f31e62..e737cb4 100644
--- a/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
+++ b/utils/usergrid-util-python/usergrid_tools/queue/dlq-iterator-checker.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from multiprocessing.pool import Pool
import argparse
import json
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py b/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
index 0b22770..080c815 100644
--- a/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
+++ b/utils/usergrid-util-python/usergrid_tools/queue/dlq_requeue.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import argparse
import json
import datetime
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py b/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
index 7f30d06..b41894a 100644
--- a/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
+++ b/utils/usergrid-util-python/usergrid_tools/queue/queue_cleaner.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import argparse
import json
import datetime
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py b/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
index 7edf1fc..01639e4 100644
--- a/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/redis/redis_iterator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from collections import defaultdict
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py b/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
index 8957861..0bf3f13 100644
--- a/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
+++ b/utils/usergrid-util-python/usergrid_tools/redis/redisscan.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import redis
r = redis.Redis("localhost", 6379)
[14/50] [abbrv] usergrid git commit: removed unnecessary NPE check..
adding it in the methods call it.
Posted by mr...@apache.org.
removed unnecessary NPE check.. adding it in the methods call it.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6a790a8e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6a790a8e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6a790a8e
Branch: refs/heads/master
Commit: 6a790a8ef44a0d64eb4833b1cecb10b68032f8b4
Parents: 897d373
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Fri Jul 8 11:38:46 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Fri Jul 8 11:38:46 2016 -0700
----------------------------------------------------------------------
.../usergrid/security/tokens/cassandra/TokenServiceImpl.java | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6a790a8e/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
index 2234257..62d454a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
@@ -799,12 +799,7 @@ public class TokenServiceImpl implements TokenService {
}
private String getExternalSSOProvider(){
- try {
return properties.getProperty(USERGRID_EXTERNAL_PROVIDER);
- }
- catch(NullPointerException e ){
- throw new IllegalArgumentException("External SSO provider is enabled but the provider name is empty");
- }
}
/**
[19/50] [abbrv] usergrid git commit: USERGRID-1306 - Usergrid .NET
SDK Integration Tests assume http://api.usergrid.com endpoint for API * Fixes
https://issues.apache.org/jira/browse/USERGRID-1306 * Add a new apiUri
configuration to the settings files th
Posted by mr...@apache.org.
USERGRID-1306 - Usergrid .NET SDK Integration Tests assume http://api.usergrid.com endpoint for API
* Fixes https://issues.apache.org/jira/browse/USERGRID-1306
* Add a new apiUri configuration to the settings files that allow for running integration tests against something aside from http://api.usergrid.com
* If nothing is specified for this config or if it is missing, the default behavior of using http://api.usergrid.com is enforced
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5a0756a5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5a0756a5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5a0756a5
Branch: refs/heads/master
Commit: 5a0756a54e796efa45d07865ea7efbad8a922f8f
Parents: 9e038ef
Author: Andrew Lane <an...@gmail.com>
Authored: Thu Jul 14 09:58:46 2016 -0400
Committer: Andrew Lane <an...@gmail.com>
Committed: Thu Jul 14 09:58:46 2016 -0400
----------------------------------------------------------------------
.../Usergrid.Sdk.IntegrationTests/BaseTest.cs | 23 +++++++++++++++++---
.../EntityPagingTests.cs | 2 +-
.../Usergrid.Sdk.IntegrationTests/GroupTests.cs | 4 ++--
.../Usergrid.Sdk.IntegrationTests/LoginTests.cs | 12 +++++-----
.../Usergrid.Sdk.IntegrationTests.dll.config | 1 +
5 files changed, 30 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5a0756a5/sdks/dotnet/Usergrid.Sdk.IntegrationTests/BaseTest.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/BaseTest.cs b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/BaseTest.cs
index 1669569..b83cf7e 100644
--- a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/BaseTest.cs
+++ b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/BaseTest.cs
@@ -32,7 +32,24 @@ namespace Usergrid.Sdk.IntegrationTests
_config = config;
}
- protected string Organization
+ /// <summary>
+ /// The URI of the Usergrid API, which defaults to api.usergrid.com if none is specified, just like the Client object does
+ /// </summary>
+ protected string ApiUri
+ {
+ get
+ {
+ var apiUri = GetAppSetting("apiUri");
+ if (String.IsNullOrWhiteSpace(apiUri))
+ {
+ apiUri = "http://api.usergrid.com";
+ }
+
+ return apiUri;
+ }
+ }
+
+ protected string Organization
{
get{ return GetAppSetting("organization");}
}
@@ -84,12 +101,12 @@ namespace Usergrid.Sdk.IntegrationTests
private string GetAppSetting(string key)
{
- return _config == null ? ConfigurationManager.AppSettings[key] : _config.AppSettings.Settings[key].Value;
+ return _config == null ? ConfigurationManager.AppSettings[key] : _config.AppSettings.Settings[key]?.Value;
}
protected IClient InitializeClientAndLogin(AuthType authType)
{
- var client = new Client(Organization, Application);
+ var client = new Client(Organization, Application, ApiUri);
if (authType == AuthType.Application || authType == AuthType.Organization)
client.Login(ClientId, ClientSecret, authType);
else if (authType == AuthType.User)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5a0756a5/sdks/dotnet/Usergrid.Sdk.IntegrationTests/EntityPagingTests.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/EntityPagingTests.cs b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/EntityPagingTests.cs
index fbf615f..79234de 100644
--- a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/EntityPagingTests.cs
+++ b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/EntityPagingTests.cs
@@ -31,7 +31,7 @@ namespace Usergrid.Sdk.IntegrationTests
[Test]
public void ShouldDoPaging()
{
- var client = new Client(Organization, Application);
+ var client = new Client(Organization, Application, ApiUri);
client.Login(ClientId, ClientSecret, AuthType.Organization);
for (var i=0; i<20; i++)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5a0756a5/sdks/dotnet/Usergrid.Sdk.IntegrationTests/GroupTests.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/GroupTests.cs b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/GroupTests.cs
index 16bfa2c..4213c24 100644
--- a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/GroupTests.cs
+++ b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/GroupTests.cs
@@ -30,7 +30,7 @@ namespace Usergrid.Sdk.IntegrationTests
[Test]
public void ShouldManageGroupLifecycle()
{
- var client = new Client(Organization, Application);
+ var client = new Client(Organization, Application, ApiUri);
client.Login(ClientId, ClientSecret, AuthType.Organization);
var group = client.GetGroup<MyUsergridGroup>("group1");
@@ -68,7 +68,7 @@ namespace Usergrid.Sdk.IntegrationTests
[Test]
public void ShouldManageUsersInGroup()
{
- var client = new Client(Organization, Application);
+ var client = new Client(Organization, Application, ApiUri);
client.Login(ClientId, ClientSecret, AuthType.Organization);
var user = SetupUsergridUser(client, new MyUsergridUser {UserName = "user1", Password = "user1", Email = "user1@gmail.com", City = "city1"});
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5a0756a5/sdks/dotnet/Usergrid.Sdk.IntegrationTests/LoginTests.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/LoginTests.cs b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/LoginTests.cs
index aa4e1de..ea2590c 100644
--- a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/LoginTests.cs
+++ b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/LoginTests.cs
@@ -24,14 +24,14 @@ namespace Usergrid.Sdk.IntegrationTests
[Test]
public void ShouldLoginSuccessfullyWithClientCredentials()
{
- var client = new Client(Organization, Application);
+ var client = new Client(Organization, Application, ApiUri);
client.Login(ClientId, ClientSecret, AuthType.Organization);
}
[Test]
public void ShouldThrowWithInvalidOrganizationCredentials()
{
- var client = new Client (Organization, Application);
+ var client = new Client (Organization, Application, ApiUri);
try
{
@@ -48,14 +48,14 @@ namespace Usergrid.Sdk.IntegrationTests
[Test]
public void ShouldLoginSuccessfullyWithApplicationCredentials()
{
- var client = new Client(Organization, Application);
+ var client = new Client(Organization, Application, ApiUri);
client.Login(ApplicationId, ApplicationSecret, AuthType.Application);
}
[Test]
public void ShouldThrowWithInvalidApplicationCredentials()
{
- var client = new Client (Organization, Application);
+ var client = new Client (Organization, Application, ApiUri);
try
{
@@ -72,14 +72,14 @@ namespace Usergrid.Sdk.IntegrationTests
[Test]
public void ShouldLoginSuccessfullyWithUserCredentials()
{
- var client = new Client(Organization, Application);
+ var client = new Client(Organization, Application, ApiUri);
client.Login(UserId, UserSecret, AuthType.User);
}
[Test]
public void ShouldThrowWithInvalidUserCredentials()
{
- var client = new Client(Organization, Application);
+ var client = new Client(Organization, Application, ApiUri);
try
{
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5a0756a5/sdks/dotnet/Usergrid.Sdk.IntegrationTests/Usergrid.Sdk.IntegrationTests.dll.config
----------------------------------------------------------------------
diff --git a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/Usergrid.Sdk.IntegrationTests.dll.config b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/Usergrid.Sdk.IntegrationTests.dll.config
index 9785ff4..e8fd2e5 100644
--- a/sdks/dotnet/Usergrid.Sdk.IntegrationTests/Usergrid.Sdk.IntegrationTests.dll.config
+++ b/sdks/dotnet/Usergrid.Sdk.IntegrationTests/Usergrid.Sdk.IntegrationTests.dll.config
@@ -18,6 +18,7 @@
<configuration>
<appSettings>
+ <add key="apiUri" value="BASE_URI_OF_USERGRID_API" />
<add key="organization" value="ORGANIZATION_NAME" />
<add key="application" value="APPLICATION_NAME" />
<add key="clientId" value="CLIENT_ID" />
[48/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/552/head' of github.com:apache/usergrid
Posted by mr...@apache.org.
Merge commit 'refs/pull/552/head' of github.com:apache/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a35cd38a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a35cd38a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a35cd38a
Branch: refs/heads/master
Commit: a35cd38a3afa7c9e47626ac058079a74ca3fa4ba
Parents: 2ba664c be18bc1
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Aug 1 09:52:06 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Aug 1 09:52:06 2016 -0700
----------------------------------------------------------------------
sdks/python/.gitignore | 57 +
sdks/python/GUIDE.md | 2 +
sdks/python/LICENSE | 202 ++
sdks/python/README.md | 16 +
sdks/python/README.rst | 20 +
sdks/python/sample_app.py | 77 +
sdks/python/setup.py | 51 +
sdks/python/usergrid/UsergridApplication.py | 65 +
sdks/python/usergrid/UsergridAuth.py | 105 +
sdks/python/usergrid/UsergridClient.py | 401 ++++
sdks/python/usergrid/UsergridCollection.py | 82 +
sdks/python/usergrid/UsergridConnection.py | 30 +
sdks/python/usergrid/UsergridError.py | 21 +
sdks/python/usergrid/UsergridOrganization.py | 35 +
sdks/python/usergrid/UsergridQueryIterator.py | 157 ++
sdks/python/usergrid/__init__.py | 37 +
sdks/python/usergrid/app_templates.py | 38 +
sdks/python/usergrid/management_templates.py | 27 +
utils/usergrid-util-python/.gitignore | 61 +
utils/usergrid-util-python/LICENSE | 202 ++
utils/usergrid-util-python/README.md | 15 +
.../es_tools/alias_mover.py | 72 +
.../es_tools/cluster_shard_allocation.py | 111 +
.../es_tools/command_sender.py | 52 +
.../es_tools/es_index_iterator_reindexer.py | 128 +
.../es_tools/es_searcher.py | 45 +
.../es_tools/index_deleter.py | 98 +
.../es_tools/index_prefix_checker.py | 100 +
.../es_tools/index_replica_setter.py | 124 +
.../es_tools/index_shard_allocator.py | 149 ++
.../es_tools/mapping_deleter.py | 53 +
.../es_tools/mapping_retriever.py | 64 +
.../es_tools/monitor_tasks.py | 61 +
utils/usergrid-util-python/index_test/README.md | 1 +
.../index_test/document_creator.py | 276 +++
.../index_test/index_test_mixed_batch.py | 552 +++++
.../index_test/index_test_single_type_batch.py | 555 +++++
utils/usergrid-util-python/requirements.txt | 4 +
.../activity_streams/activity_streams.py | 154 ++
.../samples/beacon-event-example.py | 238 ++
.../samples/counter_test.py | 52 +
utils/usergrid-util-python/setup.py | 59 +
.../usergrid_tools/__init__.py | 4 +
.../usergrid_tools/general/__init__.py | 21 +
.../usergrid_tools/general/deleter.py | 170 ++
.../general/duplicate_name_checker.py | 47 +
.../usergrid_tools/general/queue_monitor.py | 138 ++
.../usergrid_tools/general/url_tester.py | 108 +
.../general/user_confirm_activate.py | 51 +
.../usergrid_tools/groups/__init__.py | 2 +
.../usergrid_tools/groups/big_group_creater.py | 100 +
.../usergrid_tools/indexing/README.md | 22 +
.../usergrid_tools/indexing/__init__.py | 21 +
.../usergrid_tools/indexing/batch_index_test.py | 362 +++
.../indexing/entity_index_test.py | 339 +++
.../usergrid_tools/iterators/README.md | 8 +
.../usergrid_tools/iterators/__init__.py | 18 +
.../usergrid_tools/iterators/simple_iterator.py | 101 +
.../iterators/usergrid_cross_region_iterator.py | 425 ++++
.../iterators/usergrid_iterator.py | 504 ++++
.../usergrid_tools/library_check.py | 45 +
.../usergrid_tools/migration/README.md | 234 ++
.../usergrid_tools/migration/__init__.py | 24 +
.../migration/usergrid_data_exporter.py | 943 ++++++++
.../migration/usergrid_data_migrator.py | 2186 ++++++++++++++++++
.../usergrid_tools/parse_importer/README.md | 90 +
.../usergrid_tools/parse_importer/__init__.py | 21 +
.../parse_importer/parse_importer.py | 404 ++++
.../usergrid_tools/permissions/README.md | 3 +
.../usergrid_tools/permissions/permissions.py | 168 ++
.../usergrid_tools/queue/README.md | 1 +
.../queue/dlq-iterator-checker.py | 162 ++
.../usergrid_tools/queue/dlq_requeue.py | 192 ++
.../queue/queue-config-sample.json | 22 +
.../usergrid_tools/queue/queue_cleaner.py | 174 ++
.../usergrid_tools/redis/redis_iterator.py | 52 +
.../usergrid_tools/redis/redisscan.py | 37 +
77 files changed, 11848 insertions(+)
----------------------------------------------------------------------
[06/50] [abbrv] usergrid git commit: Disable new token generation on
get user details.
Posted by mr...@apache.org.
Disable new token generation on get user details.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8d79d365
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8d79d365
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8d79d365
Branch: refs/heads/master
Commit: 8d79d365c8bc2387086bdaa72fff13de205ea94e
Parents: 3764234
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 6 14:58:00 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 6 14:58:00 2016 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/rest/management/users/UserResource.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8d79d365/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index e431579..36dbdde 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -195,9 +195,9 @@ public class UserResource extends AbstractContextResource {
ApiResponse response = createApiResponse();
response.setAction( "get admin user" );
- String token = management.getAccessTokenForAdminUser( user.getUuid(), ttl );
+ //String token = management.getAccessTokenForAdminUser( user.getUuid(), ttl );
Map<String, Object> userOrganizationData = management.getAdminUserOrganizationData( user, !shallow );
- userOrganizationData.put( "token", token );
+ //userOrganizationData.put( "token", token );
response.setData( userOrganizationData );
response.setSuccess();
[04/50] [abbrv] usergrid git commit: Allow UG to inject this class
even if public key URL is not specified because external SSO is disabled.
Posted by mr...@apache.org.
Allow UG to inject this class even if public key URL is not specified because external SSO is disabled.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/37642347
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/37642347
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/37642347
Branch: refs/heads/master
Commit: 37642347b684cedd657101073f7a6b512300d83a
Parents: 71cbd8e
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Jun 27 13:38:33 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Jun 27 13:38:33 2016 -0700
----------------------------------------------------------------------
.../apache/usergrid/security/sso/ApigeeSSO2Provider.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/37642347/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
index f35e546..de10591 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/sso/ApigeeSSO2Provider.java
@@ -64,8 +64,14 @@ public class ApigeeSSO2Provider implements ExternalSSOProvider {
}
private String getPublicKey() {
- Map<String, Object> publicKey = client.target(properties.getProperty(USERGRID_EXTERNAL_PUBLICKEY_URL)).request().get(Map.class);
- return publicKey.get(RESPONSE_PUBLICKEY_VALUE).toString().split("----\n")[1].split("\n---")[0];
+
+ final String keyUrl = properties.getProperty(USERGRID_EXTERNAL_PUBLICKEY_URL);
+ if(keyUrl != null && !keyUrl.isEmpty()) {
+ Map<String, Object> publicKey = client.target(properties.getProperty(USERGRID_EXTERNAL_PUBLICKEY_URL)).request().get(Map.class);
+ return publicKey.get(RESPONSE_PUBLICKEY_VALUE).toString().split("----\n")[1].split("\n---")[0];
+ }
+
+ return null;
}
@Override
[33/50] [abbrv] usergrid git commit: changed to apache
Posted by mr...@apache.org.
changed to apache
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/180bccf1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/180bccf1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/180bccf1
Branch: refs/heads/master
Commit: 180bccf113f7145346835174792eb79497c18b19
Parents: 32f9e55
Author: Jeff West <jw...@apigee.com>
Authored: Tue Jul 26 15:33:34 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Tue Jul 26 15:33:34 2016 -0700
----------------------------------------------------------------------
utils/usergrid-util-python/LICENSE | 222 +++++++++++++++++++++++++++++---
1 file changed, 201 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/180bccf1/utils/usergrid-util-python/LICENSE
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/LICENSE b/utils/usergrid-util-python/LICENSE
index 4db993e..8f71f43 100644
--- a/utils/usergrid-util-python/LICENSE
+++ b/utils/usergrid-util-python/LICENSE
@@ -1,22 +1,202 @@
-The MIT License (MIT)
-
-Copyright (c) 2015 Jeffrey West
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
[17/50] [abbrv] usergrid git commit: for users with SSO2 enabled,
the admin users cannot reset the password.
Posted by mr...@apache.org.
for users with SSO2 enabled, the admin users cannot reset the password.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6e093bc0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6e093bc0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6e093bc0
Branch: refs/heads/master
Commit: 6e093bc0b156c371a461440784180cf0c5487118
Parents: d393445
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Tue Jul 12 11:40:18 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Tue Jul 12 11:40:18 2016 -0700
----------------------------------------------------------------------
.../usergrid/rest/management/users/UserResource.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6e093bc0/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index f568463..c66822c 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -136,6 +136,11 @@ public class UserResource extends AbstractContextResource {
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
+ throw new IllegalArgumentException( "Admin Users must reset passwords via " +
+ properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
+ }
+
if ( json == null ) {
return null;
}
@@ -474,6 +479,11 @@ public class UserResource extends AbstractContextResource {
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback,
@QueryParam( "token" ) String token ) throws Exception {
+ if ( tokens.isExternalSSOProviderEnabled() ) {
+ throw new IllegalArgumentException( "Admin Users must tokens must be revoked via " +
+ properties.getProperty( USERGRID_EXTERNAL_PROVIDER_URL ) );
+ }
+
UUID adminId = user.getUuid();
this.token = token;
[03/50] [abbrv] usergrid git commit: Merge branch
'apigee-sso-provider' of https://github.com/apache/usergrid into
SSO2-Usergrid
Posted by mr...@apache.org.
Merge branch 'apigee-sso-provider' of https://github.com/apache/usergrid into SSO2-Usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f097e35e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f097e35e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f097e35e
Branch: refs/heads/master
Commit: f097e35eecfbb62bca7344d91a6c68e057c09420
Parents: 87dad49 71cbd8e
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Mon Jun 27 00:20:31 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Mon Jun 27 00:20:31 2016 -0700
----------------------------------------------------------------------
.../cassandra/ManagementServiceImpl.java | 7 +++++++
.../usergrid/security/sso/ApigeeSSO2Provider.java | 16 ++++++++++++++++
.../usergrid/security/sso/ExternalSSOProvider.java | 16 ++++++++++++++++
.../usergrid/security/sso/SSOProviderFactory.java | 17 ++++++++++++++++-
.../security/sso/UsergridExternalProvider.java | 16 ++++++++++++++++
5 files changed, 71 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[20/50] [abbrv] usergrid git commit: Fix typo in default properties
file.
Posted by mr...@apache.org.
Fix typo in default properties file.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4e93bd61
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4e93bd61
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4e93bd61
Branch: refs/heads/master
Commit: 4e93bd6199c47a7da3de4886bced22720c947ab1
Parents: fad65a8
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jul 14 13:26:04 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jul 14 13:26:04 2016 -0700
----------------------------------------------------------------------
stack/config/src/main/resources/usergrid-default.properties | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4e93bd61/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 371b251..b296a04 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -19,7 +19,7 @@
# USERGRID DEPLOYMENT PROPERTIES
###############################################################################
#
-# Contained below are propertiefs used to configure the Usergrid application.
+# Contained below are properties used to configure the Usergrid application.
# Some of the core settings depend on the specific Usergrid deployment architecture.
# For more info, check the docs at:
# <http://usergrid.readthedocs.org/en/two-dot-o-instructions/index.html>
[07/50] [abbrv] usergrid git commit: disabling activationWorkFlow if
external SSO provider is enabled.
Posted by mr...@apache.org.
disabling activationWorkFlow if external SSO provider is enabled.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/66ca27c6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/66ca27c6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/66ca27c6
Branch: refs/heads/master
Commit: 66ca27c6fadcd322dcb9f32f3fca2fd14f6bf1d0
Parents: 526748b
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Thu Jul 7 16:34:47 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Thu Jul 7 16:34:47 2016 -0700
----------------------------------------------------------------------
.../organizations/users/UsersResource.java | 13 +++++---
.../rest/management/users/UserResource.java | 4 +++
.../rest/management/users/UsersResource.java | 9 +++++-
.../security/shiro/utils/SubjectUtils.java | 32 ++++++++------------
.../usergrid/security/tokens/TokenService.java | 10 ++++--
.../tokens/cassandra/TokenServiceImpl.java | 2 +-
6 files changed, 43 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/66ca27c6/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java
index 40ba92e..03106c0 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java
@@ -129,11 +129,16 @@ public class UsersResource extends AbstractContextResource {
}
if ( user == null ) {
- user = management.createAdminUser( organization.getUuid(), username, name, email, password, false, false );
- // A null may be returned if the user fails validation check
- if ( user != null ) {
- management.startAdminUserPasswordResetFlow( organization.getUuid(), user );
+ if ( tokens.isExternalSSOProviderEnabled() ){
+ user = management.createAdminUser(organization.getUuid(),username,name,email,password,true,false);
+ }
+ else {
+ user = management.createAdminUser(organization.getUuid(), username, name, email, password, false, false);
+ // A null may be returned if the user fails validation check
+ if (user != null) {
+ management.startAdminUserPasswordResetFlow(organization.getUuid(), user);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/66ca27c6/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index e431579..1c5bcdf 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.rest.management.users;
import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
import net.tanesha.recaptcha.ReCaptchaImpl;
import net.tanesha.recaptcha.ReCaptchaResponse;
+import org.apache.shiro.SecurityUtils;
import org.apache.usergrid.management.ActivationState;
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.rest.AbstractContextResource;
@@ -27,6 +28,7 @@ import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.management.users.organizations.OrganizationsResource;
import org.apache.usergrid.rest.security.annotations.RequireAdminUserAccess;
+import org.apache.usergrid.security.shiro.principals.PrincipalIdentifier;
import org.apache.usergrid.security.tokens.TokenInfo;
import org.apache.usergrid.security.tokens.exceptions.TokenException;
import org.apache.usergrid.services.ServiceResults;
@@ -72,6 +74,8 @@ public class UserResource extends AbstractContextResource {
public UserResource init( UserInfo user ) {
this.user = user;
+ PrincipalIdentifier userPrincipal = (PrincipalIdentifier) SecurityUtils.getSubject().getPrincipal();
+ this.token = userPrincipal.getAccessTokenCredentials().getToken();
return this;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/66ca27c6/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
index 7356124..9730e06 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
@@ -138,7 +138,14 @@ public class UsersResource extends AbstractContextResource {
ApiResponse response = createApiResponse();
response.setAction( "create user" );
- UserInfo user = management.createAdminUser( null, username, name, email, password, false, false );
+
+ UserInfo user = null;
+ if ( tokens.isExternalSSOProviderEnabled() ){
+ user = management.createAdminUser(null,username,name,email,password,true,false);
+ }
+ else {
+ user = management.createAdminUser(null, username, name, email, password, false, false);
+ }
Map<String, Object> result = new LinkedHashMap<String, Object>();
if ( user != null ) {
result.put( "user", user );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/66ca27c6/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/SubjectUtils.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/SubjectUtils.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/SubjectUtils.java
index 3b4b37d..822e2c0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/SubjectUtils.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/utils/SubjectUtils.java
@@ -17,34 +17,28 @@
package org.apache.usergrid.security.shiro.utils;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
+import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.security.shiro.PrincipalCredentialsToken;
-import org.apache.usergrid.security.shiro.principals.UserPrincipal;
-
import org.apache.commons.lang.StringUtils;
import org.apache.shiro.SecurityUtils;
import org.apache.shiro.UnavailableSecurityManagerException;
import org.apache.shiro.session.Session;
import org.apache.shiro.subject.Subject;
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.index.query.Identifier;
+import org.apache.usergrid.security.shiro.PrincipalCredentialsToken;
+import org.apache.usergrid.security.shiro.principals.UserPrincipal;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.google.common.collect.BiMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
import static org.apache.commons.lang.StringUtils.isNotBlank;
-import org.apache.usergrid.persistence.index.query.Identifier;
-import static org.apache.usergrid.security.shiro.Realm.ROLE_ADMIN_USER;
-import static org.apache.usergrid.security.shiro.Realm.ROLE_APPLICATION_ADMIN;
-import static org.apache.usergrid.security.shiro.Realm.ROLE_APPLICATION_USER;
-import static org.apache.usergrid.security.shiro.Realm.ROLE_ORGANIZATION_ADMIN;
-import static org.apache.usergrid.security.shiro.Realm.ROLE_SERVICE_ADMIN;
+import static org.apache.usergrid.security.shiro.Realm.*;
public class SubjectUtils {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/66ca27c6/stack/services/src/main/java/org/apache/usergrid/security/tokens/TokenService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/tokens/TokenService.java b/stack/services/src/main/java/org/apache/usergrid/security/tokens/TokenService.java
index 2ef5d59..308c428 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/tokens/TokenService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/tokens/TokenService.java
@@ -17,11 +17,11 @@
package org.apache.usergrid.security.tokens;
+import org.apache.usergrid.security.AuthPrincipalInfo;
+
import java.util.Map;
import java.util.UUID;
-import org.apache.usergrid.security.AuthPrincipalInfo;
-
public interface TokenService {
@@ -65,4 +65,10 @@ public interface TokenService {
* given principal uuid and application uuid
*/
public void removeTokens( AuthPrincipalInfo principal ) throws Exception;
+
+
+ /**
+ * checks if the external SSO provider is enabled.
+ */
+ public boolean isExternalSSOProviderEnabled();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/66ca27c6/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
index 7beeb3b..14d685c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
@@ -789,7 +789,7 @@ public class TokenServiceImpl implements TokenService {
}
- private boolean isExternalSSOProviderEnabled() {
+ public boolean isExternalSSOProviderEnabled() {
return Boolean.valueOf(properties.getProperty( USERGRID_EXTERNAL_SSO_ENABLED ));
}
[35/50] [abbrv] usergrid git commit: added apache header,
cleaned up some things.
Posted by mr...@apache.org.
added apache header, cleaned up some things.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a57f8650
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a57f8650
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a57f8650
Branch: refs/heads/master
Commit: a57f8650e4573d6ede650b0066821f7fae2849fe
Parents: 180bccf
Author: Jeff West <jw...@apigee.com>
Authored: Tue Jul 26 15:52:46 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Tue Jul 26 15:52:46 2016 -0700
----------------------------------------------------------------------
.../es_tools/alias_mover.py | 107 +---
.../es_tools/cluster_shard_allocation.py | 23 +-
.../es_tools/command_sender.py | 27 +-
.../es_tools/es_index_iterator_reindexer.py | 21 +-
.../es_tools/es_searcher.py | 25 +-
.../es_tools/index_deleter.py | 22 +-
.../es_tools/index_iterator_size_checker.py | 25 +-
.../es_tools/index_prefix_checker.py | 21 +-
.../es_tools/index_replica_setter.py | 57 ++-
.../es_tools/index_shard_allocator.py | 67 ++-
.../es_tools/mapping_deleter.py | 19 +
.../es_tools/mapping_retriever.py | 19 +
.../es_tools/monitor_tasks.py | 19 +
.../index_test/document_creator.py | 19 +
.../index_test/index_test_mixed_batch.py | 44 +-
.../index_test/index_test_single_type_batch.py | 41 +-
.../activity_streams/activity_streams.py | 19 +
.../samples/beacon-event-example.py | 28 +-
.../samples/counter_test.py | 24 +-
utils/usergrid-util-python/setup.py | 19 +
.../usergrid_tools/general/deleter.py | 19 +
.../general/duplicate_name_checker.py | 19 +
.../usergrid_tools/general/queue_monitor.py | 19 +
.../usergrid_tools/general/url_tester.py | 20 +-
.../general/user_confirm_activate.py | 19 +
.../usergrid_tools/general/user_creator.py | 19 +
.../usergrid_tools/groups/big_group_creater.py | 31 +-
.../usergrid_tools/indexing/batch_index_test.py | 19 +
.../indexing/entity_index_test.py | 19 +
.../usergrid_tools/iterators/simple_iterator.py | 19 +
.../iterators/usergrid_cross_region_iterator.py | 26 +-
.../iterators/usergrid_iterator.py | 504 +++++++++++++++++++
.../usergrid_tools/library_check.py | 19 +
.../migration/usergrid_data_exporter.py | 19 +
.../migration/usergrid_data_migrator.py | 23 +-
.../parse_importer/parse_importer.py | 19 +
.../usergrid_tools/permissions/permissions.py | 19 +
.../queue/dlq-iterator-checker.py | 19 +
.../usergrid_tools/queue/dlq_requeue.py | 19 +
.../usergrid_tools/queue/queue_cleaner.py | 19 +
.../usergrid_tools/redis/redis_iterator.py | 19 +
.../usergrid_tools/redis/redisscan.py | 19 +
42 files changed, 1355 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/alias_mover.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/alias_mover.py b/utils/usergrid-util-python/es_tools/alias_mover.py
index 2a8fe02..3b5caab 100644
--- a/utils/usergrid-util-python/es_tools/alias_mover.py
+++ b/utils/usergrid-util-python/es_tools/alias_mover.py
@@ -1,94 +1,35 @@
-import json
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+import json
import requests
-example_request = {
- "actions": [
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias"
- }
- }
- ]
-}
-
-cluster = 'rug000sr_euwi'
+cluster = 'cluster-1'
work = {
# 'remove': {
- # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'rug000sr_euwi_applications_3',
- # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'rug000sr_euwi_applications_3',
+ # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'applications_3',
+ # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'applications_3',
# },
'add': {
- '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'apigee-vfmplus-1-no-doc-18',
- '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'apigee-vfmplus-1-no-doc-18',
+ '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'my-index-1-no-doc-18',
+ '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'my-index-1-no-doc-18',
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
index a462124..2e0fcbd 100644
--- a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
+++ b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import time
import requests
@@ -10,8 +29,8 @@ __author__ = 'Jeff West @ ApigeeCorporation'
SHUTDOWN_NODES = True
nodes = [
- # 'res206wo',
- # 'res207wo',
+ 'elasticsearch206west',
+ 'elasticsearch207west',
]
base_url = 'http://localhost:9200'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/command_sender.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/command_sender.py b/utils/usergrid-util-python/es_tools/command_sender.py
index 92ecfff..8208e78 100644
--- a/utils/usergrid-util-python/es_tools/command_sender.py
+++ b/utils/usergrid-util-python/es_tools/command_sender.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
@@ -21,16 +40,16 @@ data = {
"move": {
"index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
"shard": 14,
- "from_node": "res018sy",
- "to_node": "res021sy"
+ "from_node": "elasticsearch018",
+ "to_node": "elasticsearch021"
}
},
{
"move": {
"index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
"shard": 12,
- "from_node": "res018sy",
- "to_node": "res009sy"
+ "from_node": "elasticsearch018",
+ "to_node": "elasticsearch009"
}
},
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
index f151fcb..85872e2 100644
--- a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
+++ b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import re
from multiprocessing.pool import Pool
@@ -5,7 +24,7 @@ import requests
# This script iterates an index and issues a PUT request for an empty string to force a reindex of the entity
-index_url_template = 'http://res013wo:9200/{index_name}/_search?size={size}&from={from_var}'
+index_url_template = 'http://elasticsearch013wo:9200/{index_name}/_search?size={size}&from={from_var}'
index_names = [
'es-index-name'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/es_searcher.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_searcher.py b/utils/usergrid-util-python/es_tools/es_searcher.py
index 55e54ef..4c7a297 100644
--- a/utils/usergrid-util-python/es_tools/es_searcher.py
+++ b/utils/usergrid-util-python/es_tools/es_searcher.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
@@ -5,12 +24,14 @@ import requests
__author__ = 'Jeff West @ ApigeeCorporation'
-url_template = 'http://localhost:9200/pea000ug_applications_2/_search'
+INDEX_NAME=''
+
+url_template = 'http://localhost:9200/%s/_search' % INDEX_NAME
request = {
"query": {
"term": {
- "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,constratus)"
+ "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,superbad)"
}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_deleter.py b/utils/usergrid-util-python/es_tools/index_deleter.py
index a697cf8..9b60006 100644
--- a/utils/usergrid-util-python/es_tools/index_deleter.py
+++ b/utils/usergrid-util-python/es_tools/index_deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import requests
import logging
@@ -23,8 +42,7 @@ print 'retrieved %s indices' % len(indices)
NUMBER_VALUE = 0
includes = [
- 'rug002sr_euwi',
- # 'rug002mr',
+ 'cluster1',
# 'b6768a08-b5d5-11e3-a495-10ddb1de66c3',
# 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
]
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
index 98d4373..03924b2 100644
--- a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
+++ b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import re
import traceback
@@ -92,7 +111,7 @@ def update_entity_fields(entity, fields):
field_value = field.get('long')
else:
- print 'WTF! %s' % json.dumps(field)
+ print 'Unexpected field type! %s' % json.dumps(field)
return entity_copy
entity_copy = update_entity_field(entity_copy, field_name, field_value)
@@ -227,10 +246,6 @@ while keep_going:
if len(app_id_find) > 0:
app_id = app_id_find[0]
- if app_id != '5f20f423-f2a8-11e4-a478-12a5923b55dc':
- print 'SKIPP APP ID: ' + app_id
- continue
-
entity_id_tmp = source.get('entityId')
entity_id_find = re_ent_id.findall(entity_id_tmp)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_prefix_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_prefix_checker.py b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
index d72ff3d..0ccb245 100644
--- a/utils/usergrid-util-python/es_tools/index_prefix_checker.py
+++ b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
@@ -1,7 +1,26 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from collections import defaultdict
import requests
-import logging
+
__author__ = 'Jeff West @ ApigeeCorporation'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_replica_setter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_replica_setter.py b/utils/usergrid-util-python/es_tools/index_replica_setter.py
index 7180fed..1214e48 100644
--- a/utils/usergrid-util-python/es_tools/index_replica_setter.py
+++ b/utils/usergrid-util-python/es_tools/index_replica_setter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from multiprocessing import Pool
import requests
import time
@@ -32,22 +51,8 @@ payload = {
"index.number_of_replicas": NUMBER_VALUE,
}
-# indices = ['usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_manual']
-
includes = [
# '70be096e-c2e1-11e4-8a55-12b4f5e28868',
- # 'b0c640af-bc6c-11e4-b078-12b4f5e28868',
- # 'e62e465e-bccc-11e4-b078-12b4f5e28868',
- # 'd82b6413-bccc-11e4-b078-12b4f5e28868',
- # '45914256-c27f-11e4-8a55-12b4f5e28868',
- # '2776a776-c27f-11e4-8a55-12b4f5e28868',
- # 'a54f878c-bc6c-11e4-b044-0e4cd56e19cd',
- # 'ed5b47ea-bccc-11e4-b078-12b4f5e28868',
- # 'bd4874ab-bccc-11e4-b044-0e4cd56e19cd',
- # '3d748996-c27f-11e4-8a55-12b4f5e28868',
- # '1daab807-c27f-11e4-8a55-12b4f5e28868',
- # 'd0c4f0da-d961-11e4-849d-12b4f5e28868',
- # '93e756ac-bc4e-11e4-92ae-12b4f5e28868',
]
excludes = [
@@ -87,18 +92,18 @@ def update_shards(index_name):
if update:
print index_name
- # url = '%s/%s/_settings' % (url_base, index)
- # print url
- #
- # response = requests.get('%s/%s/_settings' % (url_base, index))
- # settings = response.json()
- #
- # index_settings = settings[index]['settings']['index']
- #
- # current_replicas = int(index_settings.get('number_of_replicas'))
- #
- # if current_replicas == NUMBER_VALUE:
- # continue
+ url = '%s/%s/_settings' % (url_base, index)
+ print url
+
+ response = requests.get('%s/%s/_settings' % (url_base, index))
+ settings = response.json()
+
+ index_settings = settings[index]['settings']['index']
+
+ current_replicas = int(index_settings.get('number_of_replicas'))
+
+ if current_replicas == NUMBER_VALUE:
+ return
success = False
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_shard_allocator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_shard_allocator.py b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
index ecee095..d411744 100644
--- a/utils/usergrid-util-python/es_tools/index_shard_allocator.py
+++ b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from multiprocessing import Pool
@@ -11,33 +30,33 @@ __author__ = 'Jeff West @ ApigeeCorporation'
nodes_c32xl = [
- 'res000eu',
- 'res001eu',
- 'res002eu',
- 'res003eu',
- 'res004eu',
- 'res005eu',
- 'res009eu',
- 'res010eu',
- 'res011eu',
- 'res012eu',
- 'res013eu',
- 'res014eu',
+ 'elasticsearch000eu',
+ 'elasticsearch001eu',
+ 'elasticsearch002eu',
+ 'elasticsearch003eu',
+ 'elasticsearch004eu',
+ 'elasticsearch005eu',
+ 'elasticsearch009eu',
+ 'elasticsearch010eu',
+ 'elasticsearch011eu',
+ 'elasticsearch012eu',
+ 'elasticsearch013eu',
+ 'elasticsearch014eu',
]
nodes_c34xl = [
- 'res015eu',
- 'res018eu',
- 'res019eu',
- 'res020eu',
- 'res021eu',
- 'res022eu',
- 'res023eu',
- 'res024eu',
- 'res025eu',
- 'res026eu',
- 'res027eu',
- 'res028eu'
+ 'elasticsearch015eu',
+ 'elasticsearch018eu',
+ 'elasticsearch019eu',
+ 'elasticsearch020eu',
+ 'elasticsearch021eu',
+ 'elasticsearch022eu',
+ 'elasticsearch023eu',
+ 'elasticsearch024eu',
+ 'elasticsearch025eu',
+ 'elasticsearch026eu',
+ 'elasticsearch027eu',
+ 'elasticsearch028eu'
]
nodes = nodes_c34xl
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/mapping_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_deleter.py b/utils/usergrid-util-python/es_tools/mapping_deleter.py
index 74ad898..112cc20 100644
--- a/utils/usergrid-util-python/es_tools/mapping_deleter.py
+++ b/utils/usergrid-util-python/es_tools/mapping_deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/mapping_retriever.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_retriever.py b/utils/usergrid-util-python/es_tools/mapping_retriever.py
index 0da123b..29fbe11 100644
--- a/utils/usergrid-util-python/es_tools/mapping_retriever.py
+++ b/utils/usergrid-util-python/es_tools/mapping_retriever.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/monitor_tasks.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/monitor_tasks.py b/utils/usergrid-util-python/es_tools/monitor_tasks.py
index df23d49..b444322 100644
--- a/utils/usergrid-util-python/es_tools/monitor_tasks.py
+++ b/utils/usergrid-util-python/es_tools/monitor_tasks.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import datetime
import requests
import time
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/document_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/document_creator.py b/utils/usergrid-util-python/index_test/document_creator.py
index fd544c6..a43e965 100644
--- a/utils/usergrid-util-python/index_test/document_creator.py
+++ b/utils/usergrid-util-python/index_test/document_creator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from __future__ import print_function
from Queue import Empty
import json
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
index d1dd40c..3e6552e 100644
--- a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
+++ b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from multiprocessing import JoinableQueue, Process
import random
@@ -6,34 +25,19 @@ import traceback
import uuid
import time
import sys
-
import argparse
import loremipsum
import requests
from elasticsearch import Elasticsearch
-
es_hosts = [
- {'host': 'ees000wo', 'port': 9200},
- {'host': 'ees001wo', 'port': 9200},
- {'host': 'ees002wo', 'port': 9200},
- {'host': 'ees003wo', 'port': 9200},
- {'host': 'ees004wo', 'port': 9200},
- {'host': 'ees005wo', 'port': 9200},
- {'host': 'ees006wo', 'port': 9200},
- {'host': 'ees007wo', 'port': 9200},
- {'host': 'ees008wo', 'port': 9200},
- {'host': 'ees009wo', 'port': 9200},
- {'host': 'ees010wo', 'port': 9200},
- {'host': 'ees011wo', 'port': 9200},
- {'host': 'ees012wo', 'port': 9200},
- {'host': 'ees013wo', 'port': 9200},
- {'host': 'ees014wo', 'port': 9200},
- {'host': 'ees015wo', 'port': 9200},
- {'host': 'ees016wo', 'port': 9200},
- {'host': 'ees017wo', 'port': 9200}
+ {'host': 'elasticsearch000west', 'port': 9200},
+ {'host': 'elasticsearch001west', 'port': 9200},
+ {'host': 'elasticsearch002west', 'port': 9200},
+ {'host': 'elasticsearch003west', 'port': 9200}
]
+
def parse_args():
parser = argparse.ArgumentParser(description='ElasticSearch Index Test 1')
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
index e3afdc3..cc25570 100644
--- a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
+++ b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from multiprocessing import JoinableQueue, Process
import random
@@ -13,24 +32,10 @@ import requests
from elasticsearch import Elasticsearch
es_hosts = [
- {'host': 'ees000wo', 'port': 9200},
- {'host': 'ees001wo', 'port': 9200},
- {'host': 'ees002wo', 'port': 9200},
- {'host': 'ees003wo', 'port': 9200},
- {'host': 'ees004wo', 'port': 9200},
- {'host': 'ees005wo', 'port': 9200},
- {'host': 'ees006wo', 'port': 9200},
- {'host': 'ees007wo', 'port': 9200},
- {'host': 'ees008wo', 'port': 9200},
- {'host': 'ees009wo', 'port': 9200},
- {'host': 'ees010wo', 'port': 9200},
- {'host': 'ees011wo', 'port': 9200},
- {'host': 'ees012wo', 'port': 9200},
- {'host': 'ees013wo', 'port': 9200},
- {'host': 'ees014wo', 'port': 9200},
- {'host': 'ees015wo', 'port': 9200},
- {'host': 'ees016wo', 'port': 9200},
- {'host': 'ees017wo', 'port': 9200}
+ {'host': 'elasticsearch000west', 'port': 9200},
+ {'host': 'elasticsearch001west', 'port': 9200},
+ {'host': 'elasticsearch002west', 'port': 9200},
+ {'host': 'elasticsearch003west', 'port': 9200}
]
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
index ce38544..9d95bef 100644
--- a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
+++ b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
# docs page: http://docs.apigee.com/api-baas/content/creating-activity
# create user 1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/beacon-event-example.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/beacon-event-example.py b/utils/usergrid-util-python/samples/beacon-event-example.py
index fc05cdc..8863b35 100644
--- a/utils/usergrid-util-python/samples/beacon-event-example.py
+++ b/utils/usergrid-util-python/samples/beacon-event-example.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
# URL Templates for Usergrid
import json
import random
@@ -13,16 +32,11 @@ connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb
connection_create_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}"
url_data = {
- 'api_url': 'https://amer-apibaas-prod.apigee.net/appservices',
- 'org': 'jwest-samples',
+ 'api_url': 'https://usergridhost/basepath',
+ 'org': 'samples',
'app': 'event-example'
}
-url_data = {
- 'api_url': 'http://usergrid_app.cfapps-01.haas-26.pez.pivotal.io',
- 'org': 'jwest',
- 'app': 'sandbox'
-}
session = requests.Session()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/counter_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/counter_test.py b/utils/usergrid-util-python/samples/counter_test.py
index 7852b26..4e393b7 100644
--- a/utils/usergrid-util-python/samples/counter_test.py
+++ b/utils/usergrid-util-python/samples/counter_test.py
@@ -1,4 +1,22 @@
-import datetime
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import time
import json
@@ -8,7 +26,7 @@ tstamp = time.gmtime() * 1000
s = requests.Session()
-s.headers.update({'authorization': 'Bearer YWMt7AHANAKcEeaVR-EahuX8EgAAAVQ7Q56jxQjUsmhJn8rGLTth0XtRrBSIzDA'})
+s.headers.update({'authorization': 'Bearer TOKEN'})
s.headers.update({'content-type': 'application/json'})
url = 'https://host/appservices-new/usergrid/pushtest/events'
@@ -26,6 +44,6 @@ print r.status_code
time.sleep(30)
-r = s.get('https://host/appservices-new/usergrid/pushtest//counters?counter=counters.jeff.west')
+r = s.get('https://host/appservices-new/usergrid/pushtest/counters?counter=counters.jeff.west')
print r.text
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/setup.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/setup.py b/utils/usergrid-util-python/setup.py
index 1f19cb2..83ee6ae 100755
--- a/utils/usergrid-util-python/setup.py
+++ b/utils/usergrid-util-python/setup.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from setuptools import setup, find_packages
__author__ = 'Jeff West @ ApigeeCorporation'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/deleter.py b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
index 3c53cae..3e21fa4 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/deleter.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import traceback
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
index 3682d18..a40e097 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from usergrid import UsergridQueryIterator
### This iterates a collection using GRAPH and checks whether there are more than on entity with the same name
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
index 37594d1..2ad2950 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import argparse
import json
import datetime
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
index 62755df..9d999d6 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
@@ -1,6 +1,24 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import datetime
import time
-
import numpy
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
index 9b1484e..3e94468 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
index d21183d..ace64ee 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
index 268b7f8..3e88dfe 100644
--- a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
+++ b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import traceback
from multiprocessing import Pool
@@ -12,22 +31,14 @@ users = 10000
username_template = 'precisely-10k-%s'
url_data = {
- "api_url": "https://usergrid-e2e-prod.e2e.apigee.net/appservices-2-1/",
- "org": "tempgrey",
+ "api_url": "https://usergrid.net",
+ "org": "org",
"app": "sandbox",
"client_id": "",
"client_secret": "",
}
-url_data = {
- "api_url": "http://baas-ug002sr.apigee.net/",
- "org": "apigee-vfmplus",
- "app": "sandbox",
- "client_id": "",
- "client_secret": "",
-
-}
collection_url_template = "{api_url}/{org}/{app}/{collection}"
add_user_url_template = "{api_url}/{org}/{app}/groups/{group_name}/users/{uuid}"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
index 6c910dd..1ad8fbc 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
@@ -1,4 +1,23 @@
# -*- coding: utf-8 -*-
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import logging
import traceback
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
index 6f193a6..8ec73f6 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
@@ -1,4 +1,23 @@
# -*- coding: utf-8 -*-
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import logging
from multiprocessing import Pool
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
index 60cc4a0..c0f2ebd 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import logging
import sys
import uuid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
index 35933a2..e5e860f 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
@@ -1,7 +1,23 @@
-from usergrid import UsergridQuery
-
-__author__ = 'Jeff West @ ApigeeCorporation'
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+from usergrid import UsergridQuery
from Queue import Empty
import argparse
import json
@@ -12,10 +28,10 @@ from multiprocessing import Process, JoinableQueue
import datetime
import requests
import traceback
-from logging.handlers import RotatingFileHandler
-import urllib3
import urllib3.contrib.pyopenssl
+__author__ = 'Jeff West @ ApigeeCorporation'
+
urllib3.disable_warnings()
urllib3.contrib.pyopenssl.inject_into_urllib3()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
new file mode 100644
index 0000000..04328ab
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
@@ -0,0 +1,504 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
+from Queue import Empty
+import json
+import logging
+import sys
+from multiprocessing import Queue, Process
+import traceback
+from logging.handlers import RotatingFileHandler
+import time
+
+import argparse
+
+from usergrid import UsergridClient, UsergridError
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+logger = logging.getLogger('UsergridIterator')
+
+# SAMPLE CONFIG FILE for source and target
+sample_config = {
+ "endpoint": {
+ "api_url": "https://api.usergrid.com",
+ "limit": 100
+ },
+
+ "credentials": {
+ "myOrg": {
+ "client_id": "<<client_id>>",
+ "client_secret": "<<client_secret>>"
+ }
+ }
+}
+
+
+def init_logging(file_enabled=False, stdout_enabled=True):
+ root_logger = logging.getLogger()
+ root_logger.setLevel(logging.INFO)
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
+
+ log_formatter = logging.Formatter(fmt='%(asctime)s | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ if file_enabled:
+ log_file_name = './UsergridIterator.log'
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=204857600,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+
+ if stdout_enabled:
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+
+ root_logger.addHandler(stdout_logger)
+
+
+config = {}
+
+
+class Worker(Process):
+ """
+ The worker is used to perform a set of handler functions in a chain. Work is provided for the Worker thread(s) on
+ a JoinableQueue. The thread will continue until either 1) it is explicitly terminated or 2) until it does not
+ receive work on the queue after a consecutive number of attempts (max_empty_count) using the specified timeout
+ (queue_timeout)
+ """
+
+ def __init__(self,
+ queue,
+ source_client,
+ target_client,
+ max_empty_count=3,
+ queue_timeout=10,
+ function_chain=None):
+ """
+ This is an example handler function which can transform an entity. Multiple handler functions can be used to
+ process a entity. The response is an entity which will get passed to the next handler in the chain
+
+ :param queue: The queue on which to listen for work
+ :param source_client: The UsergridClient of the source Usergrid instance
+ :param target_client: The UsergridClient of the target Usergrid instance
+ :param max_empty_count: The maximum number of times for a worker to not receive work after checking the queue
+ :param queue_timeout: The timeout for waiting for work on the queue
+ :param function_chain: An array of function pointers which will be executed in array sequence, expeting the following parameters: org_name, app_name, collection_name, entity, source_client, target_client, attempts=0p
+ """
+
+ super(Worker, self).__init__()
+ logger.warning('Creating worker!')
+
+ if not function_chain:
+ function_chain = []
+
+ self.function_chain = function_chain
+ self.queue = queue
+ self.source_client = source_client
+ self.target_client = target_client
+ self.max_empty_count = max_empty_count
+ self.queue_timeout = queue_timeout
+
+ def run(self):
+ logger.info('starting run()...')
+ keep_going = True
+
+ count_processed = 0
+ count_failed = 0
+ empty_count = 0
+
+ while keep_going:
+
+ try:
+ org, app, collection_name, entity = self.queue.get(timeout=self.queue_timeout)
+
+ empty_count = 0
+ success = True
+ entity_param = entity
+
+ for handler in self.function_chain:
+
+ if entity_param is not None:
+ try:
+ entity_param = handler(org, app, collection_name, entity_param, self.source_client,
+ self.target_client)
+ except Exception, e:
+ logger.error(e)
+ print traceback.format_exc()
+ success = False
+
+ if success:
+ count_processed += 1
+ logger.info('Processed [%sth] SUCCESS app/collection/name/uuid = %s / %s / %s / %s' % (
+ count_processed, app, collection_name, entity.get('name'), entity.get('uuid')))
+ else:
+ count_failed += 1
+ logger.warning('Processed [%sth] FAILURE app/collection/name/uuid = %s / %s / %s / %s' % (
+ count_processed, app, collection_name, entity.get('name'), entity.get('uuid')))
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except Empty:
+ logger.warning(
+ 'No task received after timeout=[%s]! Empty Count=%s' % (self.queue_timeout, empty_count))
+
+ empty_count += 1
+
+ if empty_count >= self.max_empty_count:
+ logger.warning('Stopping work after empty_count=[%s]' % empty_count)
+ keep_going = False
+
+ logger.info('Worker finished!')
+
+
+def filter_entity(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+ """
+ This is an example handler function which can filter entities. Multiple handler functions can be used to
+ process a entity. The response is an entity which will get passed to the next handler in the chain
+
+ :param org_name: The org name from whence this entity came
+ :param app_name: The app name from whence this entity came
+ :param collection_name: The collection name from whence this entity came
+ :param entity: The entity retrieved from the source instance
+ :param source_client: The UsergridClient for the source Usergrid instance
+ :param target_client: The UsergridClient for the target Usergrid instance
+ :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+ :return: an entity. If response is None then the chain will stop.
+ """
+
+ # return None if you want to stop the chain (filter the entity out)
+ if 'blah' in entity_data:
+ return None
+
+ # return the entity to keep going
+ return entity_data
+
+
+def transform_entity(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+ """
+ This is an example handler function which can transform an entity. Multiple handler functions can be used to
+ process a entity. The response is an entity which will get passed to the next handler in the chain
+
+ :param org_name: The org name from whence this entity came
+ :param app_name: The app name from whence this entity came
+ :param collection_name: The collection name from whence this entity came
+ :param entity: The entity retrieved from the source instance
+ :param source_client: The UsergridClient for the source Usergrid instance
+ :param target_client: The UsergridClient for the target Usergrid instance
+ :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+ :return: an entity. If response is None then the chain will stop.
+ """
+ # this just returns the entity with no transform
+ return entity_data
+
+
+def create_new(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+ """
+ This is an example handler function which can be used to create a new entity in the target instance (based on the
+ target_client) parameter. Multiple handler functions can be used to process a entity. The response is an entity
+ which will get passed to the next handler in the chain
+
+ :param org_name: The org name from whence this entity came
+ :param app_name: The app name from whence this entity came
+ :param collection_name: The collection name from whence this entity came
+ :param entity_data: The entity retrieved from the source instance
+ :param source_client: The UsergridClient for the source Usergrid instance
+ :param target_client: The UsergridClient for the target Usergrid instance
+ :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+ :return: an entity. If response is None then the chain will stop.
+ """
+
+ attempts += 1
+
+ if 'metadata' in entity_data: entity_data.pop('metadata')
+
+ target_org = config.get('target_org')
+ target_app = config.get('app_mapping', {}).get(app_name, app_name)
+ target_collection = config.get('collection_mapping', {}).get(collection_name, collection_name)
+
+ if target_client:
+ try:
+ c = target_client.org(target_org).app(target_app).collection(target_collection)
+ e = c.entity_from_data(entity_data)
+ e.put()
+
+ except UsergridError, err:
+ logger.error(err)
+ raise err
+
+ return None
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid App/Collection Iterator')
+
+ parser.add_argument('-o', '--org',
+ help='Name of the org to migrate',
+ type=str,
+ required=True)
+
+ parser.add_argument('-a', '--app',
+ help='Multiple, name of apps to include, skip to include all',
+ default=[],
+ action='append')
+
+ parser.add_argument('-c', '--collection',
+ help='Multiple, name of collections to include, skip to include all',
+ default=[],
+ action='append')
+
+ parser.add_argument('--ql',
+ help='The Query string for processing the source collection(s)',
+ type=str,
+ default='select *')
+
+ parser.add_argument('-s', '--source_config',
+ help='The configuration of the source endpoint/org',
+ type=str,
+ default='source.json')
+
+ parser.add_argument('-d', '--target_config',
+ help='The configuration of the target endpoint/org',
+ type=str,
+ default='destination.json')
+
+ parser.add_argument('-w', '--workers',
+ help='The number of worker threads',
+ type=int,
+ default=1)
+
+ parser.add_argument('-f', '--force',
+ help='Force an update regardless of modified date',
+ type=bool,
+ default=False)
+
+ parser.add_argument('--max_empty_count',
+ help='The number of iterations for an individual worker to receive no work before stopping',
+ type=int,
+ default=3)
+
+ parser.add_argument('--queue_timeout',
+ help='The duration in seconds for an individual worker queue poll before Empty is raised',
+ type=int,
+ default=10)
+
+ parser.add_argument('--map_app',
+ help="A colon-separated string such as 'apples:oranges' which indicates to put data from the app named 'apples' from the source endpoint into app named 'oranges' in the target endpoint",
+ default=[],
+ action='append')
+
+ parser.add_argument('--map_collection',
+ help="A colon-separated string such as 'cats:dogs' which indicates to put data from collections named 'cats' from the source endpoint into a collection named 'dogs' in the target endpoint, applicable to all apps",
+ default=[],
+ action='append')
+
+ parser.add_argument('--target_org',
+ help="The org name at the Usergrid destination instance",
+ type=str)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+def init():
+ global config
+
+ config['collection_mapping'] = {}
+ config['app_mapping'] = {}
+ config['org_mapping'] = {}
+
+ with open(config.get('source_config'), 'r') as f:
+ config['source_config'] = json.load(f)
+
+ with open(config.get('target_config'), 'r') as f:
+ config['target_config'] = json.load(f)
+
+ for mapping in config.get('map_collection', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['collection_mapping'][parts[0]] = parts[1]
+ else:
+ logger.warning('Skipping malformed Collection mapping: [%s]' % mapping)
+
+ for mapping in config.get('map_app', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['app_mapping'][parts[0]] = parts[1]
+ else:
+ logger.warning('Skipping malformed App mapping: [%s]' % mapping)
+
+ for mapping in config.get('map_org', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['org_mapping'][parts[0]] = parts[1]
+ else:
+ logger.warning('Skipping Org mapping: [%s]' % mapping)
+
+ if 'source_config' in config:
+ config['source_endpoint'] = config['source_config'].get('endpoint').copy()
+ config['source_endpoint'].update(config['source_config']['credentials'][config['org']])
+
+ config['target_org'] = config['target_org'] if config['target_org'] else config['org']
+
+ if 'target_config' in config:
+ config['target_endpoint'] = config['target_config'].get('endpoint').copy()
+ config['target_endpoint'].update(config['target_config']['credentials'][config['target_org']])
+
+
+def wait_for(arr_threads, sleep_time=3):
+ """
+ This function pauses the thread until the array of threads which is provided all stop working
+
+ :param arr_threads: an array of Process objects to monitor
+ :param sleep_time: the time to sleep between evaluating the array for completion
+ :return: None
+ """
+ threads_working = 100
+
+ while threads_working > 0:
+ threads_working = 0
+
+ for t in arr_threads:
+
+ if t.is_alive():
+ threads_working += 1
+
+ if threads_working > 0:
+ logger.warn('Waiting for [%s] threads to finish...' % threads_working)
+ time.sleep(sleep_time)
+
+ logger.warn('Worker Threads finished!')
+
+
+class UsergridIterator:
+ def __init__(self):
+ pass
+
+ def get_to_work(self):
+ global config
+
+ queue = Queue()
+ logger.warning('Starting workers...')
+
+ apps_to_process = config.get('app')
+ collections_to_process = config.get('collection')
+ source_org = config['org']
+ target_org = config.get('target_org', config.get('org'))
+
+ source_client = None
+ target_client = None
+
+ try:
+ source_client = UsergridClient(api_url=config['source_endpoint']['api_url'],
+ org_name=source_org)
+ source_client.authenticate_management_client(
+ client_credentials=config['source_config']['credentials'][source_org])
+
+ except UsergridError, e:
+ logger.critical(e)
+ exit()
+
+ if 'target_endpoint' in config:
+ try:
+ target_client = UsergridClient(api_url=config['target_endpoint']['api_url'],
+ org_name=target_org)
+ target_client.authenticate_management_client(
+ client_credentials=config['target_config']['credentials'][target_org])
+
+ except UsergridError, e:
+ logger.critical(e)
+ exit()
+
+ function_chain = [filter_entity, transform_entity, create_new]
+
+ workers = [Worker(queue=queue,
+ source_client=source_client,
+ target_client=target_client,
+ function_chain=function_chain,
+ max_empty_count=config.get('max_empty_count', 3),
+ queue_timeout=config.get('queue_timeout', 10))
+
+ for x in xrange(config.get('workers', 1))]
+
+ [w.start() for w in workers]
+
+ for app in source_client.list_apps():
+
+ if len(apps_to_process) > 0 and app not in apps_to_process:
+ logger.warning('Skipping app=[%s]' % app)
+ continue
+
+ logger.warning('Processing app=[%s]' % app)
+
+ source_app = source_client.organization(source_org).application(app)
+
+ for collection_name, collection in source_app.list_collections().iteritems():
+
+ if collection_name in ['events', 'queues']:
+ logger.warning('Skipping internal collection=[%s]' % collection_name)
+ continue
+
+ if len(collections_to_process) > 0 and collection_name not in collections_to_process:
+ logger.warning('Skipping collection=[%s]' % collection_name)
+ continue
+
+ logger.warning('Processing collection=%s' % collection_name)
+
+ counter = 0
+
+ try:
+ for entity in collection.query(ql=config.get('ql'),
+ limit=config.get('source_endpoint', {}).get('limit', 100)):
+ counter += 1
+ queue.put((config.get('org'), app, collection_name, entity))
+
+ except KeyboardInterrupt:
+ [w.terminate() for w in workers]
+
+ logger.info('Publishing entities complete!')
+
+ wait_for(workers)
+
+ logger.info('All done!!')
+
+
+def main():
+ global config
+ config = parse_args()
+ init()
+
+ init_logging()
+
+ UsergridIterator().get_to_work()
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/library_check.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/library_check.py b/utils/usergrid-util-python/usergrid_tools/library_check.py
index f326987..0fc6b2f 100644
--- a/utils/usergrid-util-python/usergrid_tools/library_check.py
+++ b/utils/usergrid-util-python/usergrid_tools/library_check.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import traceback
url_data = {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
index 19d6f24..e374fbc 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import os
import uuid
from Queue import Empty
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
index 30ecc26..c99aa12 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import os
import uuid
from Queue import Empty
@@ -7,9 +26,7 @@ import logging
import sys
from multiprocessing import Queue, Process
from sets import Set
-
import time_uuid
-
import datetime
from cloghandler import ConcurrentRotatingFileHandler
import requests
@@ -17,9 +34,7 @@ import traceback
import redis
import time
from sys import platform as _platform
-
import signal
-
from requests.auth import HTTPBasicAuth
from usergrid import UsergridQueryIterator
import urllib3
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
index 3a2d864..0989401 100644
--- a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
+++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import logging
from logging.handlers import RotatingFileHandler
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
index e859843..95730d5 100644
--- a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
+++ b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from multiprocessing import Pool
[42/50] [abbrv] usergrid git commit: updated Apache license header
Posted by mr...@apache.org.
updated Apache license header
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c212f1f3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c212f1f3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c212f1f3
Branch: refs/heads/master
Commit: c212f1f35576a9948c7aa30ea575419e6cb2f40e
Parents: 04a896e
Author: Jeff West <jw...@apigee.com>
Authored: Thu Jul 28 14:09:17 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Thu Jul 28 14:09:17 2016 -0700
----------------------------------------------------------------------
.../usergrid_tools/general/__init__.py | 18 ++++++++++++++++++
.../usergrid_tools/indexing/__init__.py | 18 ++++++++++++++++++
.../usergrid_tools/iterators/__init__.py | 18 ++++++++++++++++++
.../migration/usergrid_data_migrator.py | 2 +-
.../usergrid_tools/parse_importer/__init__.py | 18 ++++++++++++++++++
5 files changed, 73 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c212f1f3/utils/usergrid-util-python/usergrid_tools/general/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/__init__.py b/utils/usergrid-util-python/usergrid_tools/general/__init__.py
index e69de29..b64a076 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/__init__.py
@@ -0,0 +1,18 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c212f1f3/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py b/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
index e69de29..b64a076 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/__init__.py
@@ -0,0 +1,18 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c212f1f3/utils/usergrid-util-python/usergrid_tools/iterators/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/__init__.py b/utils/usergrid-util-python/usergrid_tools/iterators/__init__.py
index e69de29..b64a076 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/__init__.py
@@ -0,0 +1,18 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c212f1f3/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
index 0ed0539..42a74ea 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
@@ -529,7 +529,7 @@ class CollectionWorker(Process):
if empty_count >= 2:
keep_going = False
- except Exception, e:
+ except Exception as e:
logger.exception('Error in CollectionWorker processing collection [%s]' % collection_name)
print traceback.format_exc()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c212f1f3/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
index e69de29..b64a076 100644
--- a/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
+++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/__init__.py
@@ -0,0 +1,18 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
\ No newline at end of file
[41/50] [abbrv] usergrid git commit: Merge branch 'master' into
apigee-sso-provider
Posted by mr...@apache.org.
Merge branch 'master' into apigee-sso-provider
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/98f95f90
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/98f95f90
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/98f95f90
Branch: refs/heads/master
Commit: 98f95f901e488ce6d5affceaaa13fce5e141a24d
Parents: 50eee86 1e11464
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jul 28 09:59:29 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jul 28 09:59:29 2016 -0700
----------------------------------------------------------------------
.../usergrid/java/client/query/UsergridQuery.java | 6 ++++--
.../usergrid/query/validator/ApiServerRunner.java | 18 +++---------------
2 files changed, 7 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[50/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/494/head' of github.com:apache/usergrid
Posted by mr...@apache.org.
Merge commit 'refs/pull/494/head' of github.com:apache/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1d65496c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1d65496c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1d65496c
Branch: refs/heads/master
Commit: 1d65496c0bdddab8b5523f0a08ada0d2c187524f
Parents: f299c90 3b00253
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Aug 1 09:53:05 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Aug 1 09:53:05 2016 -0700
----------------------------------------------------------------------
.../AbstractPathBasedCollectionService.java | 153 +++++++++++++++++++
.../AbstractPathBasedColllectionService.java | 153 -------------------
.../usergrid/services/assets/AssetsService.java | 4 +-
.../usergrid/services/groups/GroupsService.java | 4 +-
4 files changed, 157 insertions(+), 157 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d65496c/stack/services/src/main/java/org/apache/usergrid/services/groups/GroupsService.java
----------------------------------------------------------------------
[25/50] [abbrv] usergrid git commit: Initial checkin for Python
Utilities and SDK
Posted by mr...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32f9e55d/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
new file mode 100644
index 0000000..19d6f24
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
@@ -0,0 +1,923 @@
+import os
+import uuid
+from Queue import Empty
+import argparse
+import json
+import logging
+import sys
+from multiprocessing import Queue, Process
+import time_uuid
+
+import datetime
+from cloghandler import ConcurrentRotatingFileHandler
+import requests
+import traceback
+import time
+from sys import platform as _platform
+
+import signal
+
+from usergrid import UsergridQueryIterator
+import urllib3
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+ECID = str(uuid.uuid1())
+key_version = 'v4'
+
+logger = logging.getLogger('GraphMigrator')
+worker_logger = logging.getLogger('Worker')
+collection_worker_logger = logging.getLogger('CollectionWorker')
+error_logger = logging.getLogger('ErrorLogger')
+audit_logger = logging.getLogger('AuditLogger')
+status_logger = logging.getLogger('StatusLogger')
+
+urllib3.disable_warnings()
+
+DEFAULT_CREATE_APPS = False
+DEFAULT_RETRY_SLEEP = 10
+DEFAULT_PROCESSING_SLEEP = 1
+
+queue = Queue()
+QSIZE_OK = False
+
+try:
+ queue.qsize()
+ QSIZE_OK = True
+except:
+ pass
+
+session_source = requests.Session()
+session_target = requests.Session()
+
+
+def total_seconds(td):
+ return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10 ** 6
+
+
+def init_logging(stdout_enabled=True):
+ root_logger = logging.getLogger()
+ root_logger.setLevel(logging.getLevelName(config.get('log_level', 'INFO')))
+
+ # root_logger.setLevel(logging.WARN)
+
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.ERROR)
+ logging.getLogger('boto').setLevel(logging.ERROR)
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+
+ log_formatter = logging.Formatter(
+ fmt='%(asctime)s | ' + ECID + ' | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ root_logger.addHandler(stdout_logger)
+
+ if stdout_enabled:
+ stdout_logger.setLevel(logging.getLevelName(config.get('log_level', 'INFO')))
+
+ # base log file
+
+ log_file_name = '%s/migrator.log' % config.get('log_dir')
+
+ # ConcurrentRotatingFileHandler
+ rotating_file = ConcurrentRotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=404857600,
+ backupCount=0)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+
+ error_log_file_name = '%s/migrator_errors.log' % config.get('log_dir')
+ error_rotating_file = ConcurrentRotatingFileHandler(filename=error_log_file_name,
+ mode='a',
+ maxBytes=404857600,
+ backupCount=0)
+ error_rotating_file.setFormatter(log_formatter)
+ error_rotating_file.setLevel(logging.ERROR)
+
+ root_logger.addHandler(error_rotating_file)
+
+
+entity_name_map = {
+ 'users': 'username'
+}
+
+config = {}
+
+# URL Templates for Usergrid
+org_management_app_url_template = "{api_url}/management/organizations/{org}/applications?client_id={client_id}&client_secret={client_secret}"
+org_management_url_template = "{api_url}/management/organizations/{org}/applications?client_id={client_id}&client_secret={client_secret}"
+org_url_template = "{api_url}/{org}?client_id={client_id}&client_secret={client_secret}"
+app_url_template = "{api_url}/{org}/{app}?client_id={client_id}&client_secret={client_secret}"
+collection_url_template = "{api_url}/{org}/{app}/{collection}?client_id={client_id}&client_secret={client_secret}"
+collection_query_url_template = "{api_url}/{org}/{app}/{collection}?ql={ql}&client_id={client_id}&client_secret={client_secret}&limit={limit}"
+collection_graph_url_template = "{api_url}/{org}/{app}/{collection}?client_id={client_id}&client_secret={client_secret}&limit={limit}"
+connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}?client_id={client_id}&client_secret={client_secret}"
+connecting_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/connecting/{verb}?client_id={client_id}&client_secret={client_secret}"
+connection_create_by_uuid_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}?client_id={client_id}&client_secret={client_secret}"
+connection_create_by_name_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_type}/{target_name}?client_id={client_id}&client_secret={client_secret}"
+get_entity_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}&connections=none"
+get_entity_url_with_connections_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}"
+put_entity_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}?client_id={client_id}&client_secret={client_secret}"
+
+user_credentials_url_template = "{api_url}/{org}/{app}/users/{uuid}/credentials"
+
+ignore_collections = ['activities', 'queues', 'events', 'notifications']
+
+
+class StatusListener(Process):
+ def __init__(self, status_queue, worker_queue):
+ super(StatusListener, self).__init__()
+ self.status_queue = status_queue
+ self.worker_queue = worker_queue
+
+ def run(self):
+ keep_going = True
+
+ org_results = {
+ 'name': config.get('org'),
+ 'apps': {},
+ }
+
+ empty_count = 0
+
+ while keep_going:
+
+ try:
+ app, collection, status_map = self.status_queue.get(timeout=60)
+ status_logger.info('Received status update for app/collection: [%s / %s]' % (app, collection))
+ empty_count = 0
+ org_results['summary'] = {
+ 'max_created': -1,
+ 'max_modified': -1,
+ 'min_created': 1584946416000,
+ 'min_modified': 1584946416000,
+ 'count': 0,
+ 'bytes': 0
+ }
+
+ if app not in org_results['apps']:
+ org_results['apps'][app] = {
+ 'collections': {}
+ }
+
+ org_results['apps'][app]['collections'].update(status_map)
+
+ try:
+ for app, app_data in org_results['apps'].iteritems():
+ app_data['summary'] = {
+ 'max_created': -1,
+ 'max_modified': -1,
+ 'min_created': 1584946416000,
+ 'min_modified': 1584946416000,
+ 'count': 0,
+ 'bytes': 0
+ }
+
+ if 'collections' in app_data:
+ for collection, collection_data in app_data['collections'].iteritems():
+
+ app_data['summary']['count'] += collection_data['count']
+ app_data['summary']['bytes'] += collection_data['bytes']
+
+ org_results['summary']['count'] += collection_data['count']
+ org_results['summary']['bytes'] += collection_data['bytes']
+
+ # APP
+ if collection_data.get('max_modified') > app_data['summary']['max_modified']:
+ app_data['summary']['max_modified'] = collection_data.get('max_modified')
+
+ if collection_data.get('min_modified') < app_data['summary']['min_modified']:
+ app_data['summary']['min_modified'] = collection_data.get('min_modified')
+
+ if collection_data.get('max_created') > app_data['summary']['max_created']:
+ app_data['summary']['max_created'] = collection_data.get('max_created')
+
+ if collection_data.get('min_created') < app_data['summary']['min_created']:
+ app_data['summary']['min_created'] = collection_data.get('min_created')
+
+ # ORG
+ if collection_data.get('max_modified') > org_results['summary']['max_modified']:
+ org_results['summary']['max_modified'] = collection_data.get('max_modified')
+
+ if collection_data.get('min_modified') < org_results['summary']['min_modified']:
+ org_results['summary']['min_modified'] = collection_data.get('min_modified')
+
+ if collection_data.get('max_created') > org_results['summary']['max_created']:
+ org_results['summary']['max_created'] = collection_data.get('max_created')
+
+ if collection_data.get('min_created') < org_results['summary']['min_created']:
+ org_results['summary']['min_created'] = collection_data.get('min_created')
+
+ if QSIZE_OK:
+ status_logger.warn('CURRENT Queue Depth: %s' % self.worker_queue.qsize())
+
+ status_logger.warn('UPDATED status of org processed: %s' % json.dumps(org_results))
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except:
+ print traceback.format_exc()
+
+ except KeyboardInterrupt, e:
+ status_logger.warn('FINAL status of org processed: %s' % json.dumps(org_results))
+ raise e
+
+ except Empty:
+ if QSIZE_OK:
+ status_logger.warn('CURRENT Queue Depth: %s' % self.worker_queue.qsize())
+
+ status_logger.warn('CURRENT status of org processed: %s' % json.dumps(org_results))
+
+ status_logger.warning('EMPTY! Count=%s' % empty_count)
+
+ empty_count += 1
+
+ if empty_count >= 120:
+ keep_going = False
+
+ except:
+ print traceback.format_exc()
+
+ logger.warn('FINAL status of org processed: %s' % json.dumps(org_results))
+
+
+class EntityExportWorker(Process):
+ def __init__(self, work_queue, response_queue):
+ super(EntityExportWorker, self).__init__()
+ collection_worker_logger.debug('Creating worker!')
+ self.work_queue = work_queue
+ self.response_queue = response_queue
+
+ def run(self):
+
+ collection_worker_logger.info('starting run()...')
+ keep_going = True
+
+ empty_count = 0
+ app = 'NOT SET'
+ collection_name = 'NOT SET'
+ status_map = {}
+ entity_file = None
+
+ try:
+ while keep_going:
+
+ try:
+ app, collection_name = self.work_queue.get(timeout=30)
+ empty_count = 0
+
+ status_map = self.process_collection(app, collection_name)
+
+ status_map[collection_name]['iteration_finished'] = str(datetime.datetime.now())
+
+ collection_worker_logger.warning(
+ 'Collection [%s / %s / %s] loop complete! Max Created entity %s' % (
+ config.get('org'), app, collection_name, status_map[collection_name]['max_created']))
+
+ collection_worker_logger.warning(
+ 'Sending FINAL stats for app/collection [%s / %s]: %s' % (app, collection_name, status_map))
+
+ self.response_queue.put((app, collection_name, status_map))
+
+ collection_worker_logger.info('Done! Finished app/collection: %s / %s' % (app, collection_name))
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except Empty:
+ collection_worker_logger.warning('EMPTY! Count=%s' % empty_count)
+
+ empty_count += 1
+
+ if empty_count >= 2:
+ keep_going = False
+
+ except Exception, e:
+ logger.exception('Error in CollectionWorker processing collection [%s]' % collection_name)
+ print traceback.format_exc()
+
+ finally:
+ if entity_file is not None:
+ entity_file.close()
+
+ self.response_queue.put((app, collection_name, status_map))
+ collection_worker_logger.info('FINISHED!')
+
+ def process_collection(self, app, collection_name):
+
+ status_map = {
+ collection_name: {
+ 'iteration_started': str(datetime.datetime.now()),
+ 'max_created': -1,
+ 'max_modified': -1,
+ 'min_created': 1584946416000,
+ 'min_modified': 1584946416000,
+ 'count': 0,
+ 'bytes': 0
+ }
+ }
+
+ # added a flag for using graph vs query/index
+ if config.get('graph', False):
+ source_collection_url = collection_graph_url_template.format(org=config.get('org'),
+ app=app,
+ collection=collection_name,
+ limit=config.get('limit'),
+ **config.get('source_endpoint'))
+ else:
+ source_collection_url = collection_query_url_template.format(org=config.get('org'),
+ app=app,
+ collection=collection_name,
+ limit=config.get('limit'),
+ ql="select * %s" % config.get(
+ 'ql'),
+ **config.get('source_endpoint'))
+ counter = 0
+
+ # use the UsergridQuery from the Python SDK to iterate the collection
+ q = UsergridQueryIterator(source_collection_url,
+ page_delay=config.get('page_sleep_time'),
+ sleep_time=config.get('error_retry_sleep'))
+
+ directory = os.path.join(config['export_path'], ECID, config['org'], app)
+
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+
+ entity_filename = '_'.join([collection_name, 'entity-data'])
+ entity_filename_base = os.path.join(directory, entity_filename)
+ entity_file_number = 0
+ entity_file_counter = 0
+ entity_filename = '%s-%s.txt' % (entity_filename_base, entity_file_number)
+ entity_file = open(entity_filename, 'w')
+
+ edge_filename = '_'.join([collection_name, 'edge-data'])
+ edge_filename_base = os.path.join(directory, edge_filename)
+ edge_file_number = 0
+ edge_file_counter = 0
+ edge_filename = '%s-%s.txt' % (edge_filename_base, edge_file_number)
+ edge_file = open(edge_filename, 'w')
+
+ try:
+
+ for entity in q:
+ try:
+ entity_file_counter += 1
+ counter += 1
+
+ if entity_file_counter > config['entities_per_file']:
+ entity_file.close()
+ entity_file_number += 1
+ entity_file_counter = 0
+ entity_filename = '%s-%s.txt' % (entity_filename_base, entity_file_number)
+ entity_file = open(entity_filename, 'w')
+
+ entity_file.write('%s\n' % json.dumps(entity))
+
+ edge_names = get_edge_names(entity)
+
+ for edge_name in edge_names:
+ if not include_edge(collection_name, edge_name):
+ continue
+
+ connection_query_url = connection_query_url_template.format(
+ org=config.get('org'),
+ app=app,
+ verb=edge_name,
+ collection=collection_name,
+ uuid=entity.get('uuid'),
+ limit=config.get('limit'),
+ **config.get('source_endpoint'))
+
+ connection_query = UsergridQueryIterator(connection_query_url,
+ sleep_time=config.get('error_retry_sleep'))
+
+ target_uuids = []
+
+ try:
+ for target_entity in connection_query:
+ target_uuids.append(target_entity.get('uuid'))
+ except:
+ logger.exception('Error processing edge [%s] of entity [ %s / %s / %s]' % (
+ edge_name, app, collection_name, entity.get('uuid')))
+
+ if len(target_uuids) > 0:
+ edge_file_counter += 1
+
+ edges = {
+ 'entity': {
+ 'type': entity.get('type'),
+ 'uuid': entity.get('uuid')
+ },
+ 'edge_name': edge_name,
+ 'target_uuids': target_uuids
+ }
+
+ if entity_file_counter > config['entities_per_file']:
+ edge_file.close()
+ edge_file_number += 1
+ edge_file_counter = 0
+ edge_filename = '%s-%s.txt' % (edge_filename_base, edge_file_number)
+ edge_file = open(edge_filename, 'w')
+
+ edge_file.write('%s\n' % json.dumps(edges))
+
+ if 'created' in entity:
+
+ try:
+ entity_created = long(entity.get('created'))
+
+ if entity_created > status_map[collection_name]['max_created']:
+ status_map[collection_name]['max_created'] = entity_created
+ status_map[collection_name]['max_created_str'] = str(
+ datetime.datetime.fromtimestamp(entity_created / 1000))
+
+ if entity_created < status_map[collection_name]['min_created']:
+ status_map[collection_name]['min_created'] = entity_created
+ status_map[collection_name]['min_created_str'] = str(
+ datetime.datetime.fromtimestamp(entity_created / 1000))
+
+ except ValueError:
+ pass
+
+ if 'modified' in entity:
+
+ try:
+ entity_modified = long(entity.get('modified'))
+
+ if entity_modified > status_map[collection_name]['max_modified']:
+ status_map[collection_name]['max_modified'] = entity_modified
+ status_map[collection_name]['max_modified_str'] = str(
+ datetime.datetime.fromtimestamp(entity_modified / 1000))
+
+ if entity_modified < status_map[collection_name]['min_modified']:
+ status_map[collection_name]['min_modified'] = entity_modified
+ status_map[collection_name]['min_modified_str'] = str(
+ datetime.datetime.fromtimestamp(entity_modified / 1000))
+
+ except ValueError:
+ pass
+
+ status_map[collection_name]['bytes'] += count_bytes(entity)
+ status_map[collection_name]['count'] += 1
+
+ if counter % 1000 == 1:
+ try:
+ collection_worker_logger.warning(
+ 'Sending incremental stats for app/collection [%s / %s]: %s' % (
+ app, collection_name, status_map))
+
+ self.response_queue.put((app, collection_name, status_map))
+
+ if QSIZE_OK:
+ collection_worker_logger.info(
+ 'Counter=%s, collection queue depth=%s' % (
+ counter, self.work_queue.qsize()))
+ except:
+ pass
+
+ collection_worker_logger.warn(
+ 'Current status of collections processed: %s' % json.dumps(status_map))
+ except KeyboardInterrupt:
+ raise
+
+ except:
+ logger.exception(
+ 'Error processing entity %s / %s / %s' % (app, collection_name, entity.get('uuid')))
+
+ except KeyboardInterrupt:
+ raise
+
+ except:
+ logger.exception('Error processing collection %s / %s ' % (app, collection_name))
+
+ finally:
+ if edge_file is not None:
+ edge_file.close()
+
+ if entity_file is not None:
+ entity_file.close()
+
+ return status_map
+
+
+def use_name_for_collection(collection_name):
+ return collection_name in config.get('use_name_for_collection', [])
+
+
+def include_edge(collection_name, edge_name):
+ include_edges = config.get('include_edge', [])
+
+ if include_edges is None:
+ include_edges = []
+
+ exclude_edges = config.get('exclude_edge', [])
+
+ if exclude_edges is None:
+ exclude_edges = []
+
+ if len(include_edges) > 0 and edge_name not in include_edges:
+ logger.debug(
+ 'Skipping edge [%s] since it is not in INCLUDED list: %s' % (edge_name, include_edges))
+ return False
+
+ if edge_name in exclude_edges:
+ logger.debug(
+ 'Skipping edge [%s] since it is in EXCLUDED list: %s' % (edge_name, exclude_edges))
+ return False
+
+ if (collection_name in ['users', 'user'] and edge_name in ['roles', 'followers', 'groups',
+ 'feed', 'activities']) \
+ or (collection_name in ['device', 'devices'] and edge_name in ['users']) \
+ or (collection_name in ['receipts', 'receipt'] and edge_name in ['device', 'devices']):
+ # feed and activities are not retrievable...
+ # roles and groups will be more efficiently handled from the role/group -> user
+ # followers will be handled by 'following'
+ # do only this from user -> device
+ return False
+
+ return True
+
+
+def get_source_identifier(source_entity):
+ entity_type = source_entity.get('type')
+
+ source_identifier = source_entity.get('uuid')
+
+ if use_name_for_collection(entity_type):
+
+ if entity_type in ['user']:
+ source_identifier = source_entity.get('username')
+ else:
+ source_identifier = source_entity.get('name')
+
+ if source_identifier is None:
+ source_identifier = source_entity.get('uuid')
+ logger.warn('Using UUID for entity [%s / %s]' % (entity_type, source_identifier))
+
+ return source_identifier
+
+
+def include_collection(collection_name):
+ exclude = config.get('exclude_collection', [])
+
+ if exclude is not None and collection_name in exclude:
+ return False
+
+ return True
+
+
+def get_edge_names(entity):
+ out_edge_names = [edge_name for edge_name in entity.get('metadata', {}).get('collections', [])]
+ out_edge_names += [edge_name for edge_name in entity.get('metadata', {}).get('connections', [])]
+
+ return out_edge_names
+
+
+def get_uuid_time(the_uuid_string):
+ return time_uuid.TimeUUID(the_uuid_string).get_datetime()
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid Org/App Migrator')
+
+ parser.add_argument('--log_dir',
+ help='path to the place where logs will be written',
+ default='./',
+ type=str,
+ required=False)
+
+ parser.add_argument('--log_level',
+ help='log level - DEBUG, INFO, WARN, ERROR, CRITICAL',
+ default='INFO',
+ type=str,
+ required=False)
+
+ parser.add_argument('-o', '--org',
+ help='Name of the org to migrate',
+ type=str,
+ required=True)
+
+ parser.add_argument('-a', '--app',
+ help='Name of one or more apps to include, specify none to include all apps',
+ required=False,
+ action='append')
+
+ parser.add_argument('-e', '--include_edge',
+ help='Name of one or more edges/connection types to INCLUDE, specify none to include all edges',
+ required=False,
+ action='append')
+
+ parser.add_argument('--exclude_edge',
+ help='Name of one or more edges/connection types to EXCLUDE, specify none to include all edges',
+ required=False,
+ action='append')
+
+ parser.add_argument('--exclude_collection',
+ help='Name of one or more collections to EXCLUDE, specify none to include all collections',
+ required=False,
+ action='append')
+
+ parser.add_argument('-c', '--collection',
+ help='Name of one or more collections to include, specify none to include all collections',
+ default=[],
+ action='append')
+
+ parser.add_argument('-s', '--source_config',
+ help='The path to the source endpoint/org configuration file',
+ type=str,
+ default='source.json')
+
+ parser.add_argument('--export_path',
+ help='The path to save the export files',
+ type=str,
+ default='.')
+
+ parser.add_argument('--limit',
+ help='The number of entities to return per query request',
+ type=int,
+ default=100)
+
+ parser.add_argument('--entities_per_file',
+ help='The number of entities to put in one JSON file',
+ type=int,
+ default=10000)
+
+ parser.add_argument('--error_retry_sleep',
+ help='The number of seconds to wait between retrieving after an error',
+ type=float,
+ default=30)
+
+ parser.add_argument('--page_sleep_time',
+ help='The number of seconds to wait between retrieving pages from the UsergridQueryIterator',
+ type=float,
+ default=.5)
+
+ parser.add_argument('--entity_sleep_time',
+ help='The number of seconds to wait between retrieving pages from the UsergridQueryIterator',
+ type=float,
+ default=.1)
+
+ parser.add_argument('--workers',
+ dest='collection_workers',
+ help='The number of worker processes to do the migration',
+ type=int,
+ default=4)
+
+ parser.add_argument('--queue_size_max',
+ help='The max size of entities to allow in the queue',
+ type=int,
+ default=100000)
+
+ parser.add_argument('--ql',
+ help='The QL to use in the filter for reading data from collections',
+ type=str,
+ default='select * order by created asc')
+ # default='select * order by created asc')
+
+ parser.add_argument('--nohup',
+ help='specifies not to use stdout for logging',
+ action='store_true')
+
+ parser.add_argument('--graph',
+ help='Use GRAPH instead of Query',
+ dest='graph',
+ action='store_true')
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+def init():
+ global config
+
+ config['collection_mapping'] = {}
+ config['app_mapping'] = {}
+ config['org_mapping'] = {}
+
+ with open(config.get('source_config'), 'r') as f:
+ config['source_config'] = json.load(f)
+
+ if config['exclude_collection'] is None:
+ config['exclude_collection'] = []
+
+ config['source_endpoint'] = config['source_config'].get('endpoint').copy()
+ config['source_endpoint'].update(config['source_config']['credentials'][config['org']])
+
+
+def wait_for(threads, label, sleep_time=60):
+ wait = True
+
+ logger.info('Starting to wait for [%s] threads with sleep time=[%s]' % (len(threads), sleep_time))
+
+ while wait:
+ wait = False
+ alive_count = 0
+
+ for t in threads:
+
+ if t.is_alive():
+ alive_count += 1
+ logger.info('Thread [%s] is still alive' % t.name)
+
+ if alive_count > 0:
+ wait = True
+ logger.info('Continuing to wait for [%s] threads with sleep time=[%s]' % (alive_count, sleep_time))
+ time.sleep(sleep_time)
+
+ logger.warn('All workers [%s] done!' % label)
+
+
+def count_bytes(entity):
+ entity_copy = entity.copy()
+
+ if 'metadata' in entity_copy:
+ del entity_copy['metadata']
+
+ entity_str = json.dumps(entity_copy)
+
+ return len(entity_str)
+
+
+def check_response_status(r, url, exit_on_error=True):
+ if r.status_code != 200:
+ logger.critical('HTTP [%s] on URL=[%s]' % (r.status_code, url))
+ logger.critical('Response: %s' % r.text)
+
+ if exit_on_error:
+ exit()
+
+
+def main():
+ global config
+
+ config = parse_args()
+ init()
+ init_logging()
+
+ status_map = {}
+
+ org_apps = {
+ }
+
+ if len(org_apps) == 0:
+ source_org_mgmt_url = org_management_url_template.format(org=config.get('org'),
+ limit=config.get('limit'),
+ **config.get('source_endpoint'))
+
+ print 'Retrieving apps from [%s]' % source_org_mgmt_url
+ logger.info('Retrieving apps from [%s]' % source_org_mgmt_url)
+
+ try:
+ # list the apps for the SOURCE org
+ logger.info('GET %s' % source_org_mgmt_url)
+ r = session_source.get(source_org_mgmt_url)
+
+ if r.status_code != 200:
+ logger.critical(
+ 'Abort processing: Unable to retrieve apps from [%s]: %s' % (source_org_mgmt_url, r.text))
+ exit()
+
+ logger.info(json.dumps(r.text))
+
+ org_apps = r.json().get('data')
+
+ except Exception, e:
+ logger.exception('ERROR Retrieving apps from [%s]' % source_org_mgmt_url)
+ print traceback.format_exc()
+ logger.critical('Unable to retrieve apps from [%s] and will exit' % source_org_mgmt_url)
+ exit()
+
+ if _platform == "linux" or _platform == "linux2":
+ collection_queue = Queue(maxsize=config.get('queue_size_max'))
+ collection_response_queue = Queue(maxsize=config.get('queue_size_max'))
+ else:
+ collection_queue = Queue()
+ collection_response_queue = Queue()
+
+ logger.info('Starting entity_workers...')
+
+ status_listener = StatusListener(collection_response_queue, collection_queue)
+ status_listener.start()
+
+ # start the worker processes which will iterate the collections
+ collection_workers = [EntityExportWorker(collection_queue, collection_response_queue) for x in
+ xrange(config.get('collection_workers'))]
+ [w.start() for w in collection_workers]
+
+ try:
+ apps_to_process = config.get('app')
+ collections_to_process = config.get('collection')
+
+ # iterate the apps retrieved from the org
+ for org_app in sorted(org_apps.keys()):
+ logger.info('Found SOURCE App: %s' % org_app)
+
+ time.sleep(3)
+
+ for org_app in sorted(org_apps.keys()):
+ parts = org_app.split('/')
+ app = parts[1]
+
+ # if apps are specified and the current app is not in the list, skip it
+ if apps_to_process and len(apps_to_process) > 0 and app not in apps_to_process:
+ logger.warning('Skipping app [%s] not included in process list [%s]' % (app, apps_to_process))
+ continue
+
+ logger.info('Processing app=[%s]' % app)
+
+ status_map[app] = {
+ 'iteration_started': str(datetime.datetime.now()),
+ 'max_created': -1,
+ 'max_modified': -1,
+ 'min_created': 1584946416000,
+ 'min_modified': 1584946416000,
+ 'count': 0,
+ 'bytes': 0,
+ 'collections': {}
+ }
+
+ # get the list of collections from the source org/app
+ source_app_url = app_url_template.format(org=config.get('org'),
+ app=app,
+ **config.get('source_endpoint'))
+ logger.info('GET %s' % source_app_url)
+
+ r_collections = session_source.get(source_app_url)
+
+ collection_attempts = 0
+
+ # sometimes this call was not working so I put it in a loop to force it...
+ while r_collections.status_code != 200 and collection_attempts < 5:
+ collection_attempts += 1
+ logger.warning('FAILED: GET (%s) [%s] URL: %s' % (r_collections.elapsed, r_collections.status_code,
+ source_app_url))
+ time.sleep(DEFAULT_RETRY_SLEEP)
+ r_collections = session_source.get(source_app_url)
+
+ if collection_attempts >= 5:
+ logger.critical('Unable to get collections at URL %s, skipping app' % source_app_url)
+ continue
+
+ app_response = r_collections.json()
+
+ logger.info('App Response: ' + json.dumps(app_response))
+
+ app_entities = app_response.get('entities', [])
+
+ if len(app_entities) > 0:
+ app_entity = app_entities[0]
+ collections = app_entity.get('metadata', {}).get('collections', {})
+ logger.info('Collection List: %s' % collections)
+
+ # iterate the collections which are returned.
+ for collection_name, collection_data in collections.iteritems():
+ exclude_collections = config.get('exclude_collection', [])
+
+ if exclude_collections is None:
+ exclude_collections = []
+
+ # filter out collections as configured...
+ if collection_name in ignore_collections \
+ or (len(collections_to_process) > 0 and collection_name not in collections_to_process) \
+ or (len(exclude_collections) > 0 and collection_name in exclude_collections) \
+ or (config.get('migrate') == 'credentials' and collection_name != 'users'):
+ logger.warning('Skipping collection=[%s]' % collection_name)
+
+ continue
+
+ logger.info('Publishing app / collection: %s / %s' % (app, collection_name))
+
+ collection_queue.put((app, collection_name))
+
+ status_map[app]['iteration_finished'] = str(datetime.datetime.now())
+
+ logger.info('Finished publishing collections for app [%s] !' % app)
+
+ # allow collection workers to finish
+ wait_for(collection_workers, label='collection_workers', sleep_time=30)
+
+ status_listener.terminate()
+
+ except KeyboardInterrupt:
+ logger.warning('Keyboard Interrupt, aborting...')
+ collection_queue.close()
+ collection_response_queue.close()
+
+ [os.kill(super(EntityExportWorker, p).pid, signal.SIGINT) for p in collection_workers]
+ os.kill(super(StatusListener, status_listener).pid, signal.SIGINT)
+
+ [w.terminate() for w in collection_workers]
+ status_listener.terminate()
+
+ logger.info('entity_workers DONE!')
+
+
+if __name__ == "__main__":
+ main()
[30/50] [abbrv] usergrid git commit: Merge commit
'refs/pull/551/head' of github.com:apache/usergrid into apigee-sso-provider
Posted by mr...@apache.org.
Merge commit 'refs/pull/551/head' of github.com:apache/usergrid into apigee-sso-provider
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a22dc921
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a22dc921
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a22dc921
Branch: refs/heads/master
Commit: a22dc921fa21b2e5f16a09e4a18cdc60a4080079
Parents: 4e93bd6 f8d5713
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Jul 26 14:10:00 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Jul 26 14:10:00 2016 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +
stack/rest/pom.xml | 8 +
.../usergrid/rest/AbstractContextResource.java | 16 ++
.../rest/management/ManagementResource.java | 49 +++---
.../organizations/OrganizationsResource.java | 18 +-
.../rest/management/ExternalSSOEnabledIT.java | 175 +++++++++++++++++++
.../rest/management/ManagementResourceIT.java | 24 ++-
stack/services/pom.xml | 2 +-
.../cassandra/ManagementServiceImpl.java | 26 ++-
.../security/sso/ApigeeSSO2Provider.java | 28 +--
.../tokens/cassandra/TokenServiceImpl.java | 3 +
11 files changed, 300 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a22dc921/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
[13/50] [abbrv] usergrid git commit: adding comments.
Posted by mr...@apache.org.
adding comments.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/897d373e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/897d373e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/897d373e
Branch: refs/heads/master
Commit: 897d373e1e7916802177f09993712dab65f91bcc
Parents: 930308d
Author: Ayesha Dastagiri <ay...@gmail.com>
Authored: Fri Jul 8 11:32:39 2016 -0700
Committer: Ayesha Dastagiri <ay...@gmail.com>
Committed: Fri Jul 8 11:32:39 2016 -0700
----------------------------------------------------------------------
.../rest/management/organizations/users/UsersResource.java | 3 ++-
.../org/apache/usergrid/rest/management/users/UsersResource.java | 2 +-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/897d373e/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java
index 03106c0..dad2c14 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/users/UsersResource.java
@@ -131,7 +131,8 @@ public class UsersResource extends AbstractContextResource {
if ( user == null ) {
if ( tokens.isExternalSSOProviderEnabled() ){
- user = management.createAdminUser(organization.getUuid(),username,name,email,password,true,false);
+ //autoactivating user, since the activation is done via the external sso provider.
+ user = management.createAdminUser(organization.getUuid(),username,name,email,password,true,false);
}
else {
user = management.createAdminUser(organization.getUuid(), username, name, email, password, false, false);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/897d373e/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
index 64281b6..828b46a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UsersResource.java
@@ -137,7 +137,7 @@ public class UsersResource extends AbstractContextResource {
UserInfo user = null;
if ( tokens.isExternalSSOProviderEnabled() ){
- //autoactivating user, since the activation
+ //autoactivating user, since the activation is done via the external sso provider.
user = management.createAdminUser(null,username,name,email,password,true,false);
}
else {