You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@steve.apache.org by ad...@apache.org on 2015/04/11 20:19:47 UTC
svn commit: r1672914 - in /steve/steve-web:
src/asf/steve/backends/elastic.py tests/test_backends_elastic.py
Author: adc
Date: Sat Apr 11 18:19:47 2015
New Revision: 1672914
URL: http://svn.apache.org/r1672914
Log:
Sketch of how to mock out AWS Elasticsearch
pre-commit-status-crumb=4ee93138-51b5-4c93-965e-85768220986e
Added:
steve/steve-web/tests/test_backends_elastic.py
Modified:
steve/steve-web/src/asf/steve/backends/elastic.py
Modified: steve/steve-web/src/asf/steve/backends/elastic.py
URL: http://svn.apache.org/viewvc/steve/steve-web/src/asf/steve/backends/elastic.py?rev=1672914&r1=1672913&r2=1672914&view=diff
==============================================================================
--- steve/steve-web/src/asf/steve/backends/elastic.py (original)
+++ steve/steve-web/src/asf/steve/backends/elastic.py Sat Apr 11 18:19:47 2015
@@ -16,7 +16,15 @@
# specific language governing permissions and limitations
# under the License.
#
-from elasticsearch import Elasticsearch
+import hashlib
+import json
+
+import elasticsearch
+
+import time
+
+
+ELASTIC_INDEX_NAME = 'steve'
class ElasticStorageBackend(object):
@@ -26,44 +34,200 @@ class ElasticStorageBackend(object):
self.uri = uri
self.secure = secure
- self.client = Elasticsearch([
+ self.client = elasticsearch.Elasticsearch([
{
'host': host,
'port': int(port),
'url_prefix': uri,
- 'use_ssl': bool(secure)
+ 'use_ssl': (secure.lower() == 'true') if isinstance(secure, basestring) else bool(secure)
},
])
- if not self.client.indices.exists('steve'):
- self.client.indices.create(index='steve', body={
+ # Check that we have a 'steve' index. If not, create it.
+ if not self.client.indices.exists(ELASTIC_INDEX_NAME):
+ self.client.indices.create(index=ELASTIC_INDEX_NAME, body={
'settings': {
'number_of_shards': 3,
'number_of_replicas': 1
}
})
- def create_election(self, eid, title, owner, monitors, starts, ends, is_open):
- """ Create an election
- :param str eid: Election ID
- :param str title: the title (name) of the election
- :param str owner: the owner of this election
- :param list monitors: email addresses to use for monitoring
- :param str starts: the start date of the election
- :param str ends: the end date of the election
- :param bool is_open: flag that indicates if election is open to the public or not
+ def document_exists(self, election, issue=None):
+ """ Does this election or issue exist?
+ :param str election: the id of the election
+ :param str issue: the name of the issue
+ :return bool: True if the election or issue exist, False otherwise
+ """
+ doc_type = 'elections'
+ eid = election
+ if issue:
+ doc_type = 'issues'
+ eid = generate_issue_id(election, issue)
+
+ return self.client.exists(index=ELASTIC_INDEX_NAME, doc_type=doc_type, id=eid)
+
+ def get_basedata(self, election):
+ """ Get base data from an election
+ :param str election: the id of the election
+ :return: the base data from an election
"""
- election_hash = hashlib.sha512('%f-stv-%s' % (time.time(), os.environ['REMOTE_ADDR'] if 'REMOTE_ADDR' in os.environ else random.randint(1, 99999999999))).hexdigest(),
+ res = self.client.get(index=ELASTIC_INDEX_NAME, doc_type='elections', id=election)
+
+ return res['_source'] if res else None
+
+ def close(self, election, reopen=False):
+ """ Mark an election as closed
+ :param str election: the id of the election
+ :param bool reopen:
+ """
+ basedata = self.get_basedata(election)
+
+ basedata['closed'] = bool(reopen)
- base_data = {
- 'id': eid,
- 'title': title,
- 'owner': owner,
- 'monitors': monitors,
- 'starts': starts,
- 'ends': ends,
- 'hash': election_hash,
- 'open': is_open
- }
+ self.client.index(index=ELASTIC_INDEX_NAME, doc_type='elections', id=election, body=basedata)
- self.client.index(index='steve', doc_type='elections', id=eid, body=base_data)
+ def issue_get(self, election, issue):
+ "Get JSON data from an issue"
+ issuedata = None
+ ihash = ""
+
+ issue_id = generate_issue_id(election, issue)
+ response = self.client.get(index=ELASTIC_INDEX_NAME, doc_type='issues', id=issue_id)
+
+ if response:
+ issuedata = response['_source']
+ ihash = hashlib.sha224(json.dumps(issuedata)).hexdigest()
+
+ return issuedata, ihash
+
+ def votes_get(self, electionID, issueID):
+ "Read votes and return as a dict"
+ res = self.client.search(index=ELASTIC_INDEX_NAME, doc_type='votes', q="election:%s AND issue:%s" % (electionID, issueID), size=9999999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ votes = {}
+ for entry in res['hits']['hits']:
+ votes[entry['_source']['key']] = entry['_source']['data']['vote']
+ return votes
+ return {}
+
+ def votes_get_raw(self, electionID, issueID):
+ "Read votes and retunn raw format"
+ res = self.client.search(index=ELASTIC_INDEX_NAME, doc_type='votes', q="election:%s AND issue:%s" % (electionID, issueID), size=9999999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ votes = []
+ for entry in res['hits']['hits']:
+ votes.append(entry['_source'])
+ return votes
+ return {}
+
+ def election_create(self, electionID, basedata):
+ "Create a new election"
+ self.client.index(index=ELASTIC_INDEX_NAME, doc_type='elections', id=electionID, body=basedata)
+
+ def election_update(self, electionID, basedata):
+ "Update an election with new data"
+ self.client.index(index=ELASTIC_INDEX_NAME, doc_type='elections', id=electionID, body=basedata)
+
+ def issue_update(self, electionID, issueID, issueData):
+ "Update an issue with new data"
+ self.client.index(index=ELASTIC_INDEX_NAME, doc_type='issues', id=hashlib.sha224(electionID + "/" + issueID).hexdigest(), body=issueData)
+
+ def issue_list(self, election):
+ "List all issues in an election"
+ issues = []
+ try:
+ res = self.client.search(index=ELASTIC_INDEX_NAME, doc_type='issues', sort="id", q="election:%s" % election, size=999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ for entry in res['hits']['hits']:
+ issues.append(entry['_source']['id'])
+ except:
+ pass # THIS IS OKAY! ES WILL FAIL IF THERE ARE NO ISSUES YET
+ return issues
+
+ def election_list(self):
+ "List all elections"
+ elections = []
+ try:
+ res = self.client.search(index=ELASTIC_INDEX_NAME, doc_type='elections', sort="id", q="*", size=99999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ for entry in res['hits']['hits']:
+ source = entry['_source']
+ elections.append(source['id'])
+ except Exception:
+ pass # THIS IS OKAY! On initial setup, this WILL fail until an election has been created
+ return elections
+
+ def vote(self, electionID, issueID, uid, vote):
+ "Casts a vote on an issue"
+ eid = hashlib.sha224(electionID + ":" + issueID + ":" + uid).hexdigest()
+ self.client.index(index=ELASTIC_INDEX_NAME,
+ doc_type='votes',
+ id=eid,
+ body={
+ 'issue': issueID,
+ 'election': electionID,
+ 'key': uid,
+ 'data': {
+ 'timestamp': time.time(),
+ 'vote': vote
+ }
+ })
+
+ def issue_delete(self, electionID, issueID):
+ "Deletes an issue if it exists"
+ self.client.delete(index=ELASTIC_INDEX_NAME, doc_type='issues', id=hashlib.sha224(electionID + "/" + issueID).hexdigest())
+
+ def issue_create(self, electionID, issueID, data):
+ "Create an issue"
+ self.client.index(index=ELASTIC_INDEX_NAME, doc_type='issues', id=hashlib.sha224(electionID + "/" + issueID).hexdigest(), body=data)
+
+ def voter_get_uid(self, electionID, votekey):
+ "Get the UID/email for a voter given the vote key hash"
+ try:
+ res = self.client.search(index=ELASTIC_INDEX_NAME, doc_type="voters", q="election:%s" % electionID, size=999999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ for entry in res['hits']['hits']:
+ voter = entry['_source']
+ if voter['hash'] == votekey:
+ return voter['uid']
+ except:
+ return False # ES Error, probably not seeded the voters doc yet
+
+ def voter_add(self, election, PID, xhash):
+ "Add a voter to the DB"
+ eid = hashlib.sha224(election + ":" + PID).hexdigest()
+ self.client.index(index=ELASTIC_INDEX_NAME,
+ doc_type="voters",
+ id=eid,
+ body={
+ 'election': election,
+ 'hash': xhash,
+ 'uid': PID
+ })
+
+ def voter_remove(self, election, UID):
+ "Remove the voter with the given UID"
+ votehash = hashlib.sha224(election + ":" + UID).hexdigest()
+ self.client.delete(index=ELASTIC_INDEX_NAME, doc_type="voters", id=votehash)
+
+ def voter_has_voted(self, election, issue, uid):
+ "Return true if the voter has voted on this issue, otherwise false"
+ eid = hashlib.sha224(election + ":" + issue + ":" + uid).hexdigest()
+ try:
+ return self.client.exists(index=ELASTIC_INDEX_NAME, doc_type='votes', id=eid)
+ except:
+ return False
+
+
+def generate_issue_id(election, issue):
+ """ Generate an issue id from an election and issue
+ :param str election:
+ :param str issue:
+ :return:
+ """
+ return hashlib.sha224(election + '/' + issue).hexdigest()
Added: steve/steve-web/tests/test_backends_elastic.py
URL: http://svn.apache.org/viewvc/steve/steve-web/tests/test_backends_elastic.py?rev=1672914&view=auto
==============================================================================
--- steve/steve-web/tests/test_backends_elastic.py (added)
+++ steve/steve-web/tests/test_backends_elastic.py Sat Apr 11 18:19:47 2015
@@ -0,0 +1,35 @@
+import mock
+
+from asf.steve.backends import elastic
+
+
+@mock.patch('elasticsearch.Elasticsearch')
+def test_init_new(mock_elasticsearch_class):
+
+ mock_elasticsearch_class.return_value = mock_elasticsearch = mock.MagicMock()
+ mock_elasticsearch.indices = mock_indices= mock.MagicMock()
+ mock_indices.exists.return_value = False
+
+ elastic.ElasticStorageBackend('pookie.com', 1234, 'http://foo.com', 'true')
+
+ calls = [mock.call([{'url_prefix': 'http://foo.com', 'use_ssl': True, 'host': 'pookie.com', 'port': 1234}])]
+ mock_elasticsearch_class.assert_has_calls(calls)
+
+ calls = [mock.call.exists('steve'),
+ mock.call.create(body={'settings': {'number_of_replicas': 1, 'number_of_shards': 3}}, index='steve')]
+ mock_indices.assert_has_calls(calls)
+
+
+@mock.patch('elasticsearch.Elasticsearch')
+def test_init_exists(mock_elasticsearch_class):
+
+ mock_elasticsearch_class.return_value = mock_elasticsearch = mock.MagicMock()
+ mock_elasticsearch.indices = mock_indices= mock.MagicMock()
+ mock_indices.exists.return_value = True
+
+ elastic.ElasticStorageBackend('pookie.com', 1234, 'http://foo.com', 'true')
+
+ calls = [mock.call([{'url_prefix': 'http://foo.com', 'use_ssl': True, 'host': 'pookie.com', 'port': 1234}])]
+ mock_elasticsearch_class.assert_has_calls(calls)
+
+ mock_indices.assert_has_calls([])