You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/08/01 16:54:10 UTC
[35/50] [abbrv] usergrid git commit: added apache header,
cleaned up some things.
added apache header, cleaned up some things.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a57f8650
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a57f8650
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a57f8650
Branch: refs/heads/master
Commit: a57f8650e4573d6ede650b0066821f7fae2849fe
Parents: 180bccf
Author: Jeff West <jw...@apigee.com>
Authored: Tue Jul 26 15:52:46 2016 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Tue Jul 26 15:52:46 2016 -0700
----------------------------------------------------------------------
.../es_tools/alias_mover.py | 107 +---
.../es_tools/cluster_shard_allocation.py | 23 +-
.../es_tools/command_sender.py | 27 +-
.../es_tools/es_index_iterator_reindexer.py | 21 +-
.../es_tools/es_searcher.py | 25 +-
.../es_tools/index_deleter.py | 22 +-
.../es_tools/index_iterator_size_checker.py | 25 +-
.../es_tools/index_prefix_checker.py | 21 +-
.../es_tools/index_replica_setter.py | 57 ++-
.../es_tools/index_shard_allocator.py | 67 ++-
.../es_tools/mapping_deleter.py | 19 +
.../es_tools/mapping_retriever.py | 19 +
.../es_tools/monitor_tasks.py | 19 +
.../index_test/document_creator.py | 19 +
.../index_test/index_test_mixed_batch.py | 44 +-
.../index_test/index_test_single_type_batch.py | 41 +-
.../activity_streams/activity_streams.py | 19 +
.../samples/beacon-event-example.py | 28 +-
.../samples/counter_test.py | 24 +-
utils/usergrid-util-python/setup.py | 19 +
.../usergrid_tools/general/deleter.py | 19 +
.../general/duplicate_name_checker.py | 19 +
.../usergrid_tools/general/queue_monitor.py | 19 +
.../usergrid_tools/general/url_tester.py | 20 +-
.../general/user_confirm_activate.py | 19 +
.../usergrid_tools/general/user_creator.py | 19 +
.../usergrid_tools/groups/big_group_creater.py | 31 +-
.../usergrid_tools/indexing/batch_index_test.py | 19 +
.../indexing/entity_index_test.py | 19 +
.../usergrid_tools/iterators/simple_iterator.py | 19 +
.../iterators/usergrid_cross_region_iterator.py | 26 +-
.../iterators/usergrid_iterator.py | 504 +++++++++++++++++++
.../usergrid_tools/library_check.py | 19 +
.../migration/usergrid_data_exporter.py | 19 +
.../migration/usergrid_data_migrator.py | 23 +-
.../parse_importer/parse_importer.py | 19 +
.../usergrid_tools/permissions/permissions.py | 19 +
.../queue/dlq-iterator-checker.py | 19 +
.../usergrid_tools/queue/dlq_requeue.py | 19 +
.../usergrid_tools/queue/queue_cleaner.py | 19 +
.../usergrid_tools/redis/redis_iterator.py | 19 +
.../usergrid_tools/redis/redisscan.py | 19 +
42 files changed, 1355 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/alias_mover.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/alias_mover.py b/utils/usergrid-util-python/es_tools/alias_mover.py
index 2a8fe02..3b5caab 100644
--- a/utils/usergrid-util-python/es_tools/alias_mover.py
+++ b/utils/usergrid-util-python/es_tools/alias_mover.py
@@ -1,94 +1,35 @@
-import json
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+import json
import requests
-example_request = {
- "actions": [
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias"
- }
- },
- {
- "remove": {
- "index": "apigee-vfmplus",
- "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_read_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_1edb82a0-f23c-11e5-bf51-0aa04517d9d9_write_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_read_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_48e5394a-f1fd-11e5-9fdc-06ae5d93d39b_write_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_read_alias"
- }
- },
- {
- "add": {
- "index": "apigee-vmplus-docvalues",
- "alias": "rug000sr_euwi_fd7ef86f-f1fb-11e5-b407-02f0703cf0bf_write_alias"
- }
- }
- ]
-}
-
-cluster = 'rug000sr_euwi'
+cluster = 'cluster-1'
work = {
# 'remove': {
- # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'rug000sr_euwi_applications_3',
- # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'rug000sr_euwi_applications_3',
+ # '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'applications_3',
+ # '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'applications_3',
# },
'add': {
- '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'apigee-vfmplus-1-no-doc-18',
- '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'apigee-vfmplus-1-no-doc-18',
+ '2dd3bf6c-02a5-11e6-8623-069e4448b365': 'my-index-1-no-doc-18',
+ '333af5b3-02a5-11e6-81cb-02fe3195fdff': 'my-index-1-no-doc-18',
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
index a462124..2e0fcbd 100644
--- a/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
+++ b/utils/usergrid-util-python/es_tools/cluster_shard_allocation.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import time
import requests
@@ -10,8 +29,8 @@ __author__ = 'Jeff West @ ApigeeCorporation'
SHUTDOWN_NODES = True
nodes = [
- # 'res206wo',
- # 'res207wo',
+ 'elasticsearch206west',
+ 'elasticsearch207west',
]
base_url = 'http://localhost:9200'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/command_sender.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/command_sender.py b/utils/usergrid-util-python/es_tools/command_sender.py
index 92ecfff..8208e78 100644
--- a/utils/usergrid-util-python/es_tools/command_sender.py
+++ b/utils/usergrid-util-python/es_tools/command_sender.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
@@ -21,16 +40,16 @@ data = {
"move": {
"index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
"shard": 14,
- "from_node": "res018sy",
- "to_node": "res021sy"
+ "from_node": "elasticsearch018",
+ "to_node": "elasticsearch021"
}
},
{
"move": {
"index": "usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_target_final",
"shard": 12,
- "from_node": "res018sy",
- "to_node": "res009sy"
+ "from_node": "elasticsearch018",
+ "to_node": "elasticsearch009"
}
},
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
index f151fcb..85872e2 100644
--- a/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
+++ b/utils/usergrid-util-python/es_tools/es_index_iterator_reindexer.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import re
from multiprocessing.pool import Pool
@@ -5,7 +24,7 @@ import requests
# This script iterates an index and issues a PUT request for an empty string to force a reindex of the entity
-index_url_template = 'http://res013wo:9200/{index_name}/_search?size={size}&from={from_var}'
+index_url_template = 'http://elasticsearch013wo:9200/{index_name}/_search?size={size}&from={from_var}'
index_names = [
'es-index-name'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/es_searcher.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/es_searcher.py b/utils/usergrid-util-python/es_tools/es_searcher.py
index 55e54ef..4c7a297 100644
--- a/utils/usergrid-util-python/es_tools/es_searcher.py
+++ b/utils/usergrid-util-python/es_tools/es_searcher.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
@@ -5,12 +24,14 @@ import requests
__author__ = 'Jeff West @ ApigeeCorporation'
-url_template = 'http://localhost:9200/pea000ug_applications_2/_search'
+INDEX_NAME=''
+
+url_template = 'http://localhost:9200/%s/_search' % INDEX_NAME
request = {
"query": {
"term": {
- "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,constratus)"
+ "entityId": "entityId(1a78d0a6-bffb-11e5-bc61-0af922a4f655,superbad)"
}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_deleter.py b/utils/usergrid-util-python/es_tools/index_deleter.py
index a697cf8..9b60006 100644
--- a/utils/usergrid-util-python/es_tools/index_deleter.py
+++ b/utils/usergrid-util-python/es_tools/index_deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import requests
import logging
@@ -23,8 +42,7 @@ print 'retrieved %s indices' % len(indices)
NUMBER_VALUE = 0
includes = [
- 'rug002sr_euwi',
- # 'rug002mr',
+ 'cluster1',
# 'b6768a08-b5d5-11e3-a495-10ddb1de66c3',
# 'b6768a08-b5d5-11e3-a495-11ddb1de66c9',
]
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
index 98d4373..03924b2 100644
--- a/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
+++ b/utils/usergrid-util-python/es_tools/index_iterator_size_checker.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import re
import traceback
@@ -92,7 +111,7 @@ def update_entity_fields(entity, fields):
field_value = field.get('long')
else:
- print 'WTF! %s' % json.dumps(field)
+ print 'Unexpected field type! %s' % json.dumps(field)
return entity_copy
entity_copy = update_entity_field(entity_copy, field_name, field_value)
@@ -227,10 +246,6 @@ while keep_going:
if len(app_id_find) > 0:
app_id = app_id_find[0]
- if app_id != '5f20f423-f2a8-11e4-a478-12a5923b55dc':
- print 'SKIPP APP ID: ' + app_id
- continue
-
entity_id_tmp = source.get('entityId')
entity_id_find = re_ent_id.findall(entity_id_tmp)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_prefix_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_prefix_checker.py b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
index d72ff3d..0ccb245 100644
--- a/utils/usergrid-util-python/es_tools/index_prefix_checker.py
+++ b/utils/usergrid-util-python/es_tools/index_prefix_checker.py
@@ -1,7 +1,26 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from collections import defaultdict
import requests
-import logging
+
__author__ = 'Jeff West @ ApigeeCorporation'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_replica_setter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_replica_setter.py b/utils/usergrid-util-python/es_tools/index_replica_setter.py
index 7180fed..1214e48 100644
--- a/utils/usergrid-util-python/es_tools/index_replica_setter.py
+++ b/utils/usergrid-util-python/es_tools/index_replica_setter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from multiprocessing import Pool
import requests
import time
@@ -32,22 +51,8 @@ payload = {
"index.number_of_replicas": NUMBER_VALUE,
}
-# indices = ['usergrid__a34ad389-b626-11e4-848f-06b49118d7d0__application_manual']
-
includes = [
# '70be096e-c2e1-11e4-8a55-12b4f5e28868',
- # 'b0c640af-bc6c-11e4-b078-12b4f5e28868',
- # 'e62e465e-bccc-11e4-b078-12b4f5e28868',
- # 'd82b6413-bccc-11e4-b078-12b4f5e28868',
- # '45914256-c27f-11e4-8a55-12b4f5e28868',
- # '2776a776-c27f-11e4-8a55-12b4f5e28868',
- # 'a54f878c-bc6c-11e4-b044-0e4cd56e19cd',
- # 'ed5b47ea-bccc-11e4-b078-12b4f5e28868',
- # 'bd4874ab-bccc-11e4-b044-0e4cd56e19cd',
- # '3d748996-c27f-11e4-8a55-12b4f5e28868',
- # '1daab807-c27f-11e4-8a55-12b4f5e28868',
- # 'd0c4f0da-d961-11e4-849d-12b4f5e28868',
- # '93e756ac-bc4e-11e4-92ae-12b4f5e28868',
]
excludes = [
@@ -87,18 +92,18 @@ def update_shards(index_name):
if update:
print index_name
- # url = '%s/%s/_settings' % (url_base, index)
- # print url
- #
- # response = requests.get('%s/%s/_settings' % (url_base, index))
- # settings = response.json()
- #
- # index_settings = settings[index]['settings']['index']
- #
- # current_replicas = int(index_settings.get('number_of_replicas'))
- #
- # if current_replicas == NUMBER_VALUE:
- # continue
+ url = '%s/%s/_settings' % (url_base, index)
+ print url
+
+ response = requests.get('%s/%s/_settings' % (url_base, index))
+ settings = response.json()
+
+ index_settings = settings[index]['settings']['index']
+
+ current_replicas = int(index_settings.get('number_of_replicas'))
+
+ if current_replicas == NUMBER_VALUE:
+ return
success = False
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/index_shard_allocator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/index_shard_allocator.py b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
index ecee095..d411744 100644
--- a/utils/usergrid-util-python/es_tools/index_shard_allocator.py
+++ b/utils/usergrid-util-python/es_tools/index_shard_allocator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from multiprocessing import Pool
@@ -11,33 +30,33 @@ __author__ = 'Jeff West @ ApigeeCorporation'
nodes_c32xl = [
- 'res000eu',
- 'res001eu',
- 'res002eu',
- 'res003eu',
- 'res004eu',
- 'res005eu',
- 'res009eu',
- 'res010eu',
- 'res011eu',
- 'res012eu',
- 'res013eu',
- 'res014eu',
+ 'elasticsearch000eu',
+ 'elasticsearch001eu',
+ 'elasticsearch002eu',
+ 'elasticsearch003eu',
+ 'elasticsearch004eu',
+ 'elasticsearch005eu',
+ 'elasticsearch009eu',
+ 'elasticsearch010eu',
+ 'elasticsearch011eu',
+ 'elasticsearch012eu',
+ 'elasticsearch013eu',
+ 'elasticsearch014eu',
]
nodes_c34xl = [
- 'res015eu',
- 'res018eu',
- 'res019eu',
- 'res020eu',
- 'res021eu',
- 'res022eu',
- 'res023eu',
- 'res024eu',
- 'res025eu',
- 'res026eu',
- 'res027eu',
- 'res028eu'
+ 'elasticsearch015eu',
+ 'elasticsearch018eu',
+ 'elasticsearch019eu',
+ 'elasticsearch020eu',
+ 'elasticsearch021eu',
+ 'elasticsearch022eu',
+ 'elasticsearch023eu',
+ 'elasticsearch024eu',
+ 'elasticsearch025eu',
+ 'elasticsearch026eu',
+ 'elasticsearch027eu',
+ 'elasticsearch028eu'
]
nodes = nodes_c34xl
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/mapping_deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_deleter.py b/utils/usergrid-util-python/es_tools/mapping_deleter.py
index 74ad898..112cc20 100644
--- a/utils/usergrid-util-python/es_tools/mapping_deleter.py
+++ b/utils/usergrid-util-python/es_tools/mapping_deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/mapping_retriever.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/mapping_retriever.py b/utils/usergrid-util-python/es_tools/mapping_retriever.py
index 0da123b..29fbe11 100644
--- a/utils/usergrid-util-python/es_tools/mapping_retriever.py
+++ b/utils/usergrid-util-python/es_tools/mapping_retriever.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/es_tools/monitor_tasks.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/es_tools/monitor_tasks.py b/utils/usergrid-util-python/es_tools/monitor_tasks.py
index df23d49..b444322 100644
--- a/utils/usergrid-util-python/es_tools/monitor_tasks.py
+++ b/utils/usergrid-util-python/es_tools/monitor_tasks.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import datetime
import requests
import time
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/document_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/document_creator.py b/utils/usergrid-util-python/index_test/document_creator.py
index fd544c6..a43e965 100644
--- a/utils/usergrid-util-python/index_test/document_creator.py
+++ b/utils/usergrid-util-python/index_test/document_creator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from __future__ import print_function
from Queue import Empty
import json
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
index d1dd40c..3e6552e 100644
--- a/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
+++ b/utils/usergrid-util-python/index_test/index_test_mixed_batch.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from multiprocessing import JoinableQueue, Process
import random
@@ -6,34 +25,19 @@ import traceback
import uuid
import time
import sys
-
import argparse
import loremipsum
import requests
from elasticsearch import Elasticsearch
-
es_hosts = [
- {'host': 'ees000wo', 'port': 9200},
- {'host': 'ees001wo', 'port': 9200},
- {'host': 'ees002wo', 'port': 9200},
- {'host': 'ees003wo', 'port': 9200},
- {'host': 'ees004wo', 'port': 9200},
- {'host': 'ees005wo', 'port': 9200},
- {'host': 'ees006wo', 'port': 9200},
- {'host': 'ees007wo', 'port': 9200},
- {'host': 'ees008wo', 'port': 9200},
- {'host': 'ees009wo', 'port': 9200},
- {'host': 'ees010wo', 'port': 9200},
- {'host': 'ees011wo', 'port': 9200},
- {'host': 'ees012wo', 'port': 9200},
- {'host': 'ees013wo', 'port': 9200},
- {'host': 'ees014wo', 'port': 9200},
- {'host': 'ees015wo', 'port': 9200},
- {'host': 'ees016wo', 'port': 9200},
- {'host': 'ees017wo', 'port': 9200}
+ {'host': 'elasticsearch000west', 'port': 9200},
+ {'host': 'elasticsearch001west', 'port': 9200},
+ {'host': 'elasticsearch002west', 'port': 9200},
+ {'host': 'elasticsearch003west', 'port': 9200}
]
+
def parse_args():
parser = argparse.ArgumentParser(description='ElasticSearch Index Test 1')
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
index e3afdc3..cc25570 100644
--- a/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
+++ b/utils/usergrid-util-python/index_test/index_test_single_type_batch.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from multiprocessing import JoinableQueue, Process
import random
@@ -13,24 +32,10 @@ import requests
from elasticsearch import Elasticsearch
es_hosts = [
- {'host': 'ees000wo', 'port': 9200},
- {'host': 'ees001wo', 'port': 9200},
- {'host': 'ees002wo', 'port': 9200},
- {'host': 'ees003wo', 'port': 9200},
- {'host': 'ees004wo', 'port': 9200},
- {'host': 'ees005wo', 'port': 9200},
- {'host': 'ees006wo', 'port': 9200},
- {'host': 'ees007wo', 'port': 9200},
- {'host': 'ees008wo', 'port': 9200},
- {'host': 'ees009wo', 'port': 9200},
- {'host': 'ees010wo', 'port': 9200},
- {'host': 'ees011wo', 'port': 9200},
- {'host': 'ees012wo', 'port': 9200},
- {'host': 'ees013wo', 'port': 9200},
- {'host': 'ees014wo', 'port': 9200},
- {'host': 'ees015wo', 'port': 9200},
- {'host': 'ees016wo', 'port': 9200},
- {'host': 'ees017wo', 'port': 9200}
+ {'host': 'elasticsearch000west', 'port': 9200},
+ {'host': 'elasticsearch001west', 'port': 9200},
+ {'host': 'elasticsearch002west', 'port': 9200},
+ {'host': 'elasticsearch003west', 'port': 9200}
]
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
index ce38544..9d95bef 100644
--- a/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
+++ b/utils/usergrid-util-python/samples/activity_streams/activity_streams.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
# docs page: http://docs.apigee.com/api-baas/content/creating-activity
# create user 1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/beacon-event-example.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/beacon-event-example.py b/utils/usergrid-util-python/samples/beacon-event-example.py
index fc05cdc..8863b35 100644
--- a/utils/usergrid-util-python/samples/beacon-event-example.py
+++ b/utils/usergrid-util-python/samples/beacon-event-example.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
# URL Templates for Usergrid
import json
import random
@@ -13,16 +32,11 @@ connection_query_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb
connection_create_url_template = "{api_url}/{org}/{app}/{collection}/{uuid}/{verb}/{target_uuid}"
url_data = {
- 'api_url': 'https://amer-apibaas-prod.apigee.net/appservices',
- 'org': 'jwest-samples',
+ 'api_url': 'https://usergridhost/basepath',
+ 'org': 'samples',
'app': 'event-example'
}
-url_data = {
- 'api_url': 'http://usergrid_app.cfapps-01.haas-26.pez.pivotal.io',
- 'org': 'jwest',
- 'app': 'sandbox'
-}
session = requests.Session()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/samples/counter_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/samples/counter_test.py b/utils/usergrid-util-python/samples/counter_test.py
index 7852b26..4e393b7 100644
--- a/utils/usergrid-util-python/samples/counter_test.py
+++ b/utils/usergrid-util-python/samples/counter_test.py
@@ -1,4 +1,22 @@
-import datetime
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import time
import json
@@ -8,7 +26,7 @@ tstamp = time.gmtime() * 1000
s = requests.Session()
-s.headers.update({'authorization': 'Bearer YWMt7AHANAKcEeaVR-EahuX8EgAAAVQ7Q56jxQjUsmhJn8rGLTth0XtRrBSIzDA'})
+s.headers.update({'authorization': 'Bearer TOKEN'})
s.headers.update({'content-type': 'application/json'})
url = 'https://host/appservices-new/usergrid/pushtest/events'
@@ -26,6 +44,6 @@ print r.status_code
time.sleep(30)
-r = s.get('https://host/appservices-new/usergrid/pushtest//counters?counter=counters.jeff.west')
+r = s.get('https://host/appservices-new/usergrid/pushtest/counters?counter=counters.jeff.west')
print r.text
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/setup.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/setup.py b/utils/usergrid-util-python/setup.py
index 1f19cb2..83ee6ae 100755
--- a/utils/usergrid-util-python/setup.py
+++ b/utils/usergrid-util-python/setup.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from setuptools import setup, find_packages
__author__ = 'Jeff West @ ApigeeCorporation'
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/deleter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/deleter.py b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
index 3c53cae..3e21fa4 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/deleter.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/deleter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import traceback
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
index 3682d18..a40e097 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/duplicate_name_checker.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
from usergrid import UsergridQueryIterator
### This iterates a collection using GRAPH and checks whether there are more than on entity with the same name
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
index 37594d1..2ad2950 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/queue_monitor.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import argparse
import json
import datetime
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
index 62755df..9d999d6 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/url_tester.py
@@ -1,6 +1,24 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import datetime
import time
-
import numpy
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
index 9b1484e..3e94468 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_confirm_activate.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
index d21183d..ace64ee 100644
--- a/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
+++ b/utils/usergrid-util-python/usergrid_tools/general/user_creator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import requests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
index 268b7f8..3e88dfe 100644
--- a/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
+++ b/utils/usergrid-util-python/usergrid_tools/groups/big_group_creater.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import traceback
from multiprocessing import Pool
@@ -12,22 +31,14 @@ users = 10000
username_template = 'precisely-10k-%s'
url_data = {
- "api_url": "https://usergrid-e2e-prod.e2e.apigee.net/appservices-2-1/",
- "org": "tempgrey",
+ "api_url": "https://usergrid.net",
+ "org": "org",
"app": "sandbox",
"client_id": "",
"client_secret": "",
}
-url_data = {
- "api_url": "http://baas-ug002sr.apigee.net/",
- "org": "apigee-vfmplus",
- "app": "sandbox",
- "client_id": "",
- "client_secret": "",
-
-}
collection_url_template = "{api_url}/{org}/{app}/{collection}"
add_user_url_template = "{api_url}/{org}/{app}/groups/{group_name}/users/{uuid}"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
index 6c910dd..1ad8fbc 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/batch_index_test.py
@@ -1,4 +1,23 @@
# -*- coding: utf-8 -*-
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import logging
import traceback
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
index 6f193a6..8ec73f6 100644
--- a/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
+++ b/utils/usergrid-util-python/usergrid_tools/indexing/entity_index_test.py
@@ -1,4 +1,23 @@
# -*- coding: utf-8 -*-
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import logging
from multiprocessing import Pool
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
index 60cc4a0..c0f2ebd 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/simple_iterator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import logging
import sys
import uuid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
index 35933a2..e5e860f 100644
--- a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_cross_region_iterator.py
@@ -1,7 +1,23 @@
-from usergrid import UsergridQuery
-
-__author__ = 'Jeff West @ ApigeeCorporation'
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+from usergrid import UsergridQuery
from Queue import Empty
import argparse
import json
@@ -12,10 +28,10 @@ from multiprocessing import Process, JoinableQueue
import datetime
import requests
import traceback
-from logging.handlers import RotatingFileHandler
-import urllib3
import urllib3.contrib.pyopenssl
+__author__ = 'Jeff West @ ApigeeCorporation'
+
urllib3.disable_warnings()
urllib3.contrib.pyopenssl.inject_into_urllib3()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
new file mode 100644
index 0000000..04328ab
--- /dev/null
+++ b/utils/usergrid-util-python/usergrid_tools/iterators/usergrid_iterator.py
@@ -0,0 +1,504 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
+from Queue import Empty
+import json
+import logging
+import sys
+from multiprocessing import Queue, Process
+import traceback
+from logging.handlers import RotatingFileHandler
+import time
+
+import argparse
+
+from usergrid import UsergridClient, UsergridError
+
+__author__ = 'Jeff West @ ApigeeCorporation'
+
+logger = logging.getLogger('UsergridIterator')
+
+# SAMPLE CONFIG FILE for source and target
+sample_config = {
+ "endpoint": {
+ "api_url": "https://api.usergrid.com",
+ "limit": 100
+ },
+
+ "credentials": {
+ "myOrg": {
+ "client_id": "<<client_id>>",
+ "client_secret": "<<client_secret>>"
+ }
+ }
+}
+
+
+def init_logging(file_enabled=False, stdout_enabled=True):
+ root_logger = logging.getLogger()
+ root_logger.setLevel(logging.INFO)
+ logging.getLogger('urllib3.connectionpool').setLevel(logging.WARN)
+ logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.WARN)
+
+ log_formatter = logging.Formatter(fmt='%(asctime)s | %(name)s | %(processName)s | %(levelname)s | %(message)s',
+ datefmt='%m/%d/%Y %I:%M:%S %p')
+
+ if file_enabled:
+ log_file_name = './UsergridIterator.log'
+
+ rotating_file = logging.handlers.RotatingFileHandler(filename=log_file_name,
+ mode='a',
+ maxBytes=204857600,
+ backupCount=10)
+ rotating_file.setFormatter(log_formatter)
+ rotating_file.setLevel(logging.INFO)
+
+ root_logger.addHandler(rotating_file)
+
+ if stdout_enabled:
+ stdout_logger = logging.StreamHandler(sys.stdout)
+ stdout_logger.setFormatter(log_formatter)
+ stdout_logger.setLevel(logging.INFO)
+
+ root_logger.addHandler(stdout_logger)
+
+
+config = {}
+
+
+class Worker(Process):
+ """
+ The worker is used to perform a set of handler functions in a chain. Work is provided for the Worker thread(s) on
+ a JoinableQueue. The thread will continue until either 1) it is explicitly terminated or 2) until it does not
+ receive work on the queue after a consecutive number of attempts (max_empty_count) using the specified timeout
+ (queue_timeout)
+ """
+
+ def __init__(self,
+ queue,
+ source_client,
+ target_client,
+ max_empty_count=3,
+ queue_timeout=10,
+ function_chain=None):
+ """
+ This is an example handler function which can transform an entity. Multiple handler functions can be used to
+ process a entity. The response is an entity which will get passed to the next handler in the chain
+
+ :param queue: The queue on which to listen for work
+ :param source_client: The UsergridClient of the source Usergrid instance
+ :param target_client: The UsergridClient of the target Usergrid instance
+ :param max_empty_count: The maximum number of times for a worker to not receive work after checking the queue
+ :param queue_timeout: The timeout for waiting for work on the queue
+ :param function_chain: An array of function pointers which will be executed in array sequence, expeting the following parameters: org_name, app_name, collection_name, entity, source_client, target_client, attempts=0p
+ """
+
+ super(Worker, self).__init__()
+ logger.warning('Creating worker!')
+
+ if not function_chain:
+ function_chain = []
+
+ self.function_chain = function_chain
+ self.queue = queue
+ self.source_client = source_client
+ self.target_client = target_client
+ self.max_empty_count = max_empty_count
+ self.queue_timeout = queue_timeout
+
+ def run(self):
+ logger.info('starting run()...')
+ keep_going = True
+
+ count_processed = 0
+ count_failed = 0
+ empty_count = 0
+
+ while keep_going:
+
+ try:
+ org, app, collection_name, entity = self.queue.get(timeout=self.queue_timeout)
+
+ empty_count = 0
+ success = True
+ entity_param = entity
+
+ for handler in self.function_chain:
+
+ if entity_param is not None:
+ try:
+ entity_param = handler(org, app, collection_name, entity_param, self.source_client,
+ self.target_client)
+ except Exception, e:
+ logger.error(e)
+ print traceback.format_exc()
+ success = False
+
+ if success:
+ count_processed += 1
+ logger.info('Processed [%sth] SUCCESS app/collection/name/uuid = %s / %s / %s / %s' % (
+ count_processed, app, collection_name, entity.get('name'), entity.get('uuid')))
+ else:
+ count_failed += 1
+ logger.warning('Processed [%sth] FAILURE app/collection/name/uuid = %s / %s / %s / %s' % (
+ count_processed, app, collection_name, entity.get('name'), entity.get('uuid')))
+
+ except KeyboardInterrupt, e:
+ raise e
+
+ except Empty:
+ logger.warning(
+ 'No task received after timeout=[%s]! Empty Count=%s' % (self.queue_timeout, empty_count))
+
+ empty_count += 1
+
+ if empty_count >= self.max_empty_count:
+ logger.warning('Stopping work after empty_count=[%s]' % empty_count)
+ keep_going = False
+
+ logger.info('Worker finished!')
+
+
+def filter_entity(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+ """
+ This is an example handler function which can filter entities. Multiple handler functions can be used to
+ process a entity. The response is an entity which will get passed to the next handler in the chain
+
+ :param org_name: The org name from whence this entity came
+ :param app_name: The app name from whence this entity came
+ :param collection_name: The collection name from whence this entity came
+ :param entity: The entity retrieved from the source instance
+ :param source_client: The UsergridClient for the source Usergrid instance
+ :param target_client: The UsergridClient for the target Usergrid instance
+ :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+ :return: an entity. If response is None then the chain will stop.
+ """
+
+ # return None if you want to stop the chain (filter the entity out)
+ if 'blah' in entity_data:
+ return None
+
+ # return the entity to keep going
+ return entity_data
+
+
+def transform_entity(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+ """
+ This is an example handler function which can transform an entity. Multiple handler functions can be used to
+ process a entity. The response is an entity which will get passed to the next handler in the chain
+
+ :param org_name: The org name from whence this entity came
+ :param app_name: The app name from whence this entity came
+ :param collection_name: The collection name from whence this entity came
+ :param entity: The entity retrieved from the source instance
+ :param source_client: The UsergridClient for the source Usergrid instance
+ :param target_client: The UsergridClient for the target Usergrid instance
+ :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+ :return: an entity. If response is None then the chain will stop.
+ """
+ # this just returns the entity with no transform
+ return entity_data
+
+
+def create_new(org_name, app_name, collection_name, entity_data, source_client, target_client, attempts=0):
+ """
+ This is an example handler function which can be used to create a new entity in the target instance (based on the
+ target_client) parameter. Multiple handler functions can be used to process a entity. The response is an entity
+ which will get passed to the next handler in the chain
+
+ :param org_name: The org name from whence this entity came
+ :param app_name: The app name from whence this entity came
+ :param collection_name: The collection name from whence this entity came
+ :param entity_data: The entity retrieved from the source instance
+ :param source_client: The UsergridClient for the source Usergrid instance
+ :param target_client: The UsergridClient for the target Usergrid instance
+ :param attempts: the number of previous attempts this function was run (manual, not part of the framework)
+ :return: an entity. If response is None then the chain will stop.
+ """
+
+ attempts += 1
+
+ if 'metadata' in entity_data: entity_data.pop('metadata')
+
+ target_org = config.get('target_org')
+ target_app = config.get('app_mapping', {}).get(app_name, app_name)
+ target_collection = config.get('collection_mapping', {}).get(collection_name, collection_name)
+
+ if target_client:
+ try:
+ c = target_client.org(target_org).app(target_app).collection(target_collection)
+ e = c.entity_from_data(entity_data)
+ e.put()
+
+ except UsergridError, err:
+ logger.error(err)
+ raise err
+
+ return None
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description='Usergrid App/Collection Iterator')
+
+ parser.add_argument('-o', '--org',
+ help='Name of the org to migrate',
+ type=str,
+ required=True)
+
+ parser.add_argument('-a', '--app',
+ help='Multiple, name of apps to include, skip to include all',
+ default=[],
+ action='append')
+
+ parser.add_argument('-c', '--collection',
+ help='Multiple, name of collections to include, skip to include all',
+ default=[],
+ action='append')
+
+ parser.add_argument('--ql',
+ help='The Query string for processing the source collection(s)',
+ type=str,
+ default='select *')
+
+ parser.add_argument('-s', '--source_config',
+ help='The configuration of the source endpoint/org',
+ type=str,
+ default='source.json')
+
+ parser.add_argument('-d', '--target_config',
+ help='The configuration of the target endpoint/org',
+ type=str,
+ default='destination.json')
+
+ parser.add_argument('-w', '--workers',
+ help='The number of worker threads',
+ type=int,
+ default=1)
+
+ parser.add_argument('-f', '--force',
+ help='Force an update regardless of modified date',
+ type=bool,
+ default=False)
+
+ parser.add_argument('--max_empty_count',
+ help='The number of iterations for an individual worker to receive no work before stopping',
+ type=int,
+ default=3)
+
+ parser.add_argument('--queue_timeout',
+ help='The duration in seconds for an individual worker queue poll before Empty is raised',
+ type=int,
+ default=10)
+
+ parser.add_argument('--map_app',
+ help="A colon-separated string such as 'apples:oranges' which indicates to put data from the app named 'apples' from the source endpoint into app named 'oranges' in the target endpoint",
+ default=[],
+ action='append')
+
+ parser.add_argument('--map_collection',
+ help="A colon-separated string such as 'cats:dogs' which indicates to put data from collections named 'cats' from the source endpoint into a collection named 'dogs' in the target endpoint, applicable to all apps",
+ default=[],
+ action='append')
+
+ parser.add_argument('--target_org',
+ help="The org name at the Usergrid destination instance",
+ type=str)
+
+ my_args = parser.parse_args(sys.argv[1:])
+
+ return vars(my_args)
+
+
+def init():
+ global config
+
+ config['collection_mapping'] = {}
+ config['app_mapping'] = {}
+ config['org_mapping'] = {}
+
+ with open(config.get('source_config'), 'r') as f:
+ config['source_config'] = json.load(f)
+
+ with open(config.get('target_config'), 'r') as f:
+ config['target_config'] = json.load(f)
+
+ for mapping in config.get('map_collection', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['collection_mapping'][parts[0]] = parts[1]
+ else:
+ logger.warning('Skipping malformed Collection mapping: [%s]' % mapping)
+
+ for mapping in config.get('map_app', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['app_mapping'][parts[0]] = parts[1]
+ else:
+ logger.warning('Skipping malformed App mapping: [%s]' % mapping)
+
+ for mapping in config.get('map_org', []):
+ parts = mapping.split(':')
+
+ if len(parts) == 2:
+ config['org_mapping'][parts[0]] = parts[1]
+ else:
+ logger.warning('Skipping Org mapping: [%s]' % mapping)
+
+ if 'source_config' in config:
+ config['source_endpoint'] = config['source_config'].get('endpoint').copy()
+ config['source_endpoint'].update(config['source_config']['credentials'][config['org']])
+
+ config['target_org'] = config['target_org'] if config['target_org'] else config['org']
+
+ if 'target_config' in config:
+ config['target_endpoint'] = config['target_config'].get('endpoint').copy()
+ config['target_endpoint'].update(config['target_config']['credentials'][config['target_org']])
+
+
+def wait_for(arr_threads, sleep_time=3):
+ """
+ This function pauses the thread until the array of threads which is provided all stop working
+
+ :param arr_threads: an array of Process objects to monitor
+ :param sleep_time: the time to sleep between evaluating the array for completion
+ :return: None
+ """
+ threads_working = 100
+
+ while threads_working > 0:
+ threads_working = 0
+
+ for t in arr_threads:
+
+ if t.is_alive():
+ threads_working += 1
+
+ if threads_working > 0:
+ logger.warn('Waiting for [%s] threads to finish...' % threads_working)
+ time.sleep(sleep_time)
+
+ logger.warn('Worker Threads finished!')
+
+
+class UsergridIterator:
+ def __init__(self):
+ pass
+
+ def get_to_work(self):
+ global config
+
+ queue = Queue()
+ logger.warning('Starting workers...')
+
+ apps_to_process = config.get('app')
+ collections_to_process = config.get('collection')
+ source_org = config['org']
+ target_org = config.get('target_org', config.get('org'))
+
+ source_client = None
+ target_client = None
+
+ try:
+ source_client = UsergridClient(api_url=config['source_endpoint']['api_url'],
+ org_name=source_org)
+ source_client.authenticate_management_client(
+ client_credentials=config['source_config']['credentials'][source_org])
+
+ except UsergridError, e:
+ logger.critical(e)
+ exit()
+
+ if 'target_endpoint' in config:
+ try:
+ target_client = UsergridClient(api_url=config['target_endpoint']['api_url'],
+ org_name=target_org)
+ target_client.authenticate_management_client(
+ client_credentials=config['target_config']['credentials'][target_org])
+
+ except UsergridError, e:
+ logger.critical(e)
+ exit()
+
+ function_chain = [filter_entity, transform_entity, create_new]
+
+ workers = [Worker(queue=queue,
+ source_client=source_client,
+ target_client=target_client,
+ function_chain=function_chain,
+ max_empty_count=config.get('max_empty_count', 3),
+ queue_timeout=config.get('queue_timeout', 10))
+
+ for x in xrange(config.get('workers', 1))]
+
+ [w.start() for w in workers]
+
+ for app in source_client.list_apps():
+
+ if len(apps_to_process) > 0 and app not in apps_to_process:
+ logger.warning('Skipping app=[%s]' % app)
+ continue
+
+ logger.warning('Processing app=[%s]' % app)
+
+ source_app = source_client.organization(source_org).application(app)
+
+ for collection_name, collection in source_app.list_collections().iteritems():
+
+ if collection_name in ['events', 'queues']:
+ logger.warning('Skipping internal collection=[%s]' % collection_name)
+ continue
+
+ if len(collections_to_process) > 0 and collection_name not in collections_to_process:
+ logger.warning('Skipping collection=[%s]' % collection_name)
+ continue
+
+ logger.warning('Processing collection=%s' % collection_name)
+
+ counter = 0
+
+ try:
+ for entity in collection.query(ql=config.get('ql'),
+ limit=config.get('source_endpoint', {}).get('limit', 100)):
+ counter += 1
+ queue.put((config.get('org'), app, collection_name, entity))
+
+ except KeyboardInterrupt:
+ [w.terminate() for w in workers]
+
+ logger.info('Publishing entities complete!')
+
+ wait_for(workers)
+
+ logger.info('All done!!')
+
+
+def main():
+ global config
+ config = parse_args()
+ init()
+
+ init_logging()
+
+ UsergridIterator().get_to_work()
+
+
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/library_check.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/library_check.py b/utils/usergrid-util-python/usergrid_tools/library_check.py
index f326987..0fc6b2f 100644
--- a/utils/usergrid-util-python/usergrid_tools/library_check.py
+++ b/utils/usergrid-util-python/usergrid_tools/library_check.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import traceback
url_data = {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
index 19d6f24..e374fbc 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_exporter.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import os
import uuid
from Queue import Empty
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
index 30ecc26..c99aa12 100644
--- a/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
+++ b/utils/usergrid-util-python/usergrid_tools/migration/usergrid_data_migrator.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import os
import uuid
from Queue import Empty
@@ -7,9 +26,7 @@ import logging
import sys
from multiprocessing import Queue, Process
from sets import Set
-
import time_uuid
-
import datetime
from cloghandler import ConcurrentRotatingFileHandler
import requests
@@ -17,9 +34,7 @@ import traceback
import redis
import time
from sys import platform as _platform
-
import signal
-
from requests.auth import HTTPBasicAuth
from usergrid import UsergridQueryIterator
import urllib3
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
index 3a2d864..0989401 100644
--- a/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
+++ b/utils/usergrid-util-python/usergrid_tools/parse_importer/parse_importer.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
import logging
from logging.handlers import RotatingFileHandler
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a57f8650/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
----------------------------------------------------------------------
diff --git a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
index e859843..95730d5 100644
--- a/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
+++ b/utils/usergrid-util-python/usergrid_tools/permissions/permissions.py
@@ -1,3 +1,22 @@
+# */
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing,
+# * software distributed under the License is distributed on an
+# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# * KIND, either express or implied. See the License for the
+# * specific language governing permissions and limitations
+# * under the License.
+# */
+
import json
from multiprocessing import Pool