You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@steve.apache.org by hu...@apache.org on 2015/03/29 14:38:08 UTC
svn commit: r1669902 - in /steve/trunk/pysteve/lib/backends: ./ __init__.py
es.py files.py
Author: humbedooh
Date: Sun Mar 29 12:38:08 2015
New Revision: 1669902
URL: http://svn.apache.org/r1669902
Log:
Separate each DB backend into its owns script, startign with file-based and ES
Added:
steve/trunk/pysteve/lib/backends/
steve/trunk/pysteve/lib/backends/__init__.py
steve/trunk/pysteve/lib/backends/es.py
steve/trunk/pysteve/lib/backends/files.py
Added: steve/trunk/pysteve/lib/backends/__init__.py
URL: http://svn.apache.org/viewvc/steve/trunk/pysteve/lib/backends/__init__.py?rev=1669902&view=auto
==============================================================================
--- steve/trunk/pysteve/lib/backends/__init__.py (added)
+++ steve/trunk/pysteve/lib/backends/__init__.py Sun Mar 29 12:38:08 2015
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Database backends"""
+
+__all__ = ['files','es']
\ No newline at end of file
Added: steve/trunk/pysteve/lib/backends/es.py
URL: http://svn.apache.org/viewvc/steve/trunk/pysteve/lib/backends/es.py?rev=1669902&view=auto
==============================================================================
--- steve/trunk/pysteve/lib/backends/es.py (added)
+++ steve/trunk/pysteve/lib/backends/es.py Sun Mar 29 12:38:08 2015
@@ -0,0 +1,232 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import hashlib
+import json
+import os
+import random
+import time
+from lib import constants
+
+es = None
+
+def init(config):
+ global es
+ from elasticsearch import Elasticsearch
+ es = Elasticsearch([
+ {
+ 'host': config.get("elasticsearch", "host"),
+ 'port': int(config.get("elasticsearch", "port")),
+ 'url_prefix': config.get("elasticsearch", "uri"),
+ 'use_ssl': False if config.get("elasticsearch", "secure") == "false" else True
+ },
+ ])
+ if not es.indices.exists("steve"):
+ es.indices.create(index = "steve", body = {
+ "settings": {
+ "number_of_shards" : 3,
+ "number_of_replicas" : 1
+ }
+ }
+ )
+
+
+
+def exists(election, *issue):
+ doc = "elections"
+ eid = election
+ if issue and issue[0]:
+ doc = "issues"
+ eid = hashlib.sha224(election + "/" + issue[0]).hexdigest()
+ return es.exists(index="steve", doc_type=doc, id=eid)
+
+
+
+def getBasedata(election):
+ "Get base data from an election"
+ res = es.get(index="steve", doc_type="elections", id=election)
+ if res:
+ return res['_source']
+ return None
+
+
+def close(election, reopen = False):
+ "Mark an election as closed"
+ basedata = getBasedata(election)
+ if reopen:
+ basedata['closed'] = False
+ else:
+ basedata['closed'] = True
+ es.index(index="steve", doc_type="elections", id=election, body = basedata )
+
+
+def getIssue(electionID, issueID):
+ "Get JSON data from an issue"
+ issuedata = None
+ ihash = ""
+ iid = hashlib.sha224(electionID + "/" + issueID).hexdigest()
+ res = es.get(index="steve", doc_type="issues", id=iid)
+ if res:
+ issuedata = res['_source']
+ ihash = hashlib.sha224(json.dumps(issuedata)).hexdigest()
+ return issuedata, ihash
+
+
+def getVotes(electionID, issueID):
+ "Read votes from the vote file"
+ res = es.search(index="steve", doc_type="votes", q = "election:%s AND issue:%s" % (electionID, issueID), size = 9999999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ votes = {}
+ for entry in res['hits']['hits']:
+ votes[entry['_source']['key']] = entry['_source']['data']['vote']
+ return votes
+ return {}
+
+
+
+def getVotesRaw(electionID, issueID):
+ res = es.search(index="steve", doc_type="votes", q = "election:%s AND issue:%s" % (electionID, issueID), size = 9999999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ votes = {}
+ for entry in res['hits']['hits']:
+ votes[entry['_source']['key']] = entry['_source']['data']
+ return votes
+ return {}
+
+
+def createElection(electionID, basedata):
+ "Create a new election"
+ es.index(index="steve", doc_type="elections", id=electionID, body =
+ basedata
+ );
+
+def updateElection(electionID, basedata):
+ es.index(index = "steve", doc_type = "elections", id=electionID, body = basedata)
+
+def updateIssue(electionID, issueID, issueData):
+ es.index(index = "steve", doc_type = "issues", id=hashlib.sha224(electionID + "/" + issueID).hexdigest(), body = issueData)
+
+
+def listIssues(election):
+ "List all issues in an election"
+ issues = []
+ try:
+ res = es.search(index="steve", doc_type="issues", sort = "id", q = "election:%s" % election, size = 999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ for entry in res['hits']['hits']:
+ issues.append(entry['_source']['id'])
+ except:
+ pass # THIS IS OKAY! ES WILL FAIL IF THERE ARE NO ISSUES YET
+ return issues
+
+def listElections():
+ "List all elections"
+ elections = []
+ try:
+ res = es.search(index="steve", doc_type="elections", sort = "id", q = "*", size = 99999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ for entry in res['hits']['hits']:
+ source = entry['_source']
+ elections.append(source['id'])
+ except Exception as err:
+ pass # THIS IS OKAY! On initial setup, this WILL fail until an election has been created
+ return elections
+
+def vote(electionID, issueID, uid, vote):
+ "Casts a vote on an issue"
+ eid = hashlib.sha224(electionID + ":" + issueID + ":" + uid).hexdigest()
+ es.index(index="steve", doc_type="votes", id=eid, body =
+ {
+ 'issue': issueID,
+ 'election': electionID,
+ 'key': uid,
+ 'data': {
+ 'timestamp': time.time(),
+ 'vote': vote
+ }
+ }
+ );
+
+
+def deleteIssue(electionID, issueID):
+ "Deletes an issue if it exists"
+ es.delete(index="steve", doc_type="issues", id=hashlib.sha224(electionID + "/" + issueID).hexdigest());
+
+def createIssue(electionID, issueID, data):
+ es.index(index="steve", doc_type="issues", id=hashlib.sha224(electionID + "/" + issueID).hexdigest(), body = data);
+
+
+
+def voter_get(electionID, votekey):
+ "Get the UID/email for a voter given the vote key hash"
+ try:
+ res = es.search(index="steve", doc_type="voters", q = "election:%s" % electionID, size = 999999)
+ results = len(res['hits']['hits'])
+ if results > 0:
+ for entry in res['hits']['hits']:
+ voter = entry['_source']
+ if voter['hash'] == votekey:
+ return voter['uid']
+ except:
+ return False # ES Error, probably not seeded the voters doc yet
+
+def voter_add(election, PID, xhash):
+ eid = hashlib.sha224(election + ":" + PID).hexdigest()
+ es.index(index="steve", doc_type="voters", id=eid, body = {
+ 'election': election,
+ 'hash': xhash,
+ 'uid': PID
+ }
+ )
+
+def voter_remove(election, UID):
+ votehash = hashlib.sha224(election + ":" + UID).hexdigest()
+ es.delete(index="steve", doc_type="voters", id=votehash);
+
+def voter_has_voted(election, issue, uid):
+ eid = hashlib.sha224(election + ":" + issue + ":" + uid).hexdigest()
+ try:
+ return es.exists(index="steve", doc_type="votes", id=eid)
+ except:
+ return False
+
+constants.appendBackend( {
+ 'id': 'elasticsearch',
+ 'init': init,
+ 'document_exists': exists,
+ 'get_basedata': getBasedata,
+ 'election_close': close,
+ 'election_vote': vote,
+ 'election_list': listElections,
+ 'issue_list': listIssues,
+ 'election_create': createElection,
+ 'issue_create': createIssue,
+ 'issue_delete': deleteIssue,
+ 'election_update': updateElection,
+ 'issue_update': updateIssue,
+ 'issue_get': getIssue,
+ 'vote': vote,
+ 'votes_get': getVotes,
+ 'votes_get_raw': getVotesRaw,
+ 'voter_get_uid': voter_get,
+ 'voter_add': voter_add,
+ 'voter_remove': voter_remove,
+ 'voter_has_voted': voter_has_voted
+})
\ No newline at end of file
Added: steve/trunk/pysteve/lib/backends/files.py
URL: http://svn.apache.org/viewvc/steve/trunk/pysteve/lib/backends/files.py?rev=1669902&view=auto
==============================================================================
--- steve/trunk/pysteve/lib/backends/files.py (added)
+++ steve/trunk/pysteve/lib/backends/files.py Sun Mar 29 12:38:08 2015
@@ -0,0 +1,239 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import hashlib
+import json
+import os
+import random
+import time
+
+from lib import constants
+
+homedir = None
+
+def init(config):
+ global homedir
+ homedir = config.get("general", "homedir")
+
+
+def exists(election, *issue):
+ "Returns True if an election/issue exists, False otherwise"
+ elpath = os.path.join(homedir, "issues", election)
+ if issue:
+ elpath += "/" + issue[0] + ".json"
+ return os.path.isfile(elpath)
+ else:
+ return os.path.isdir(elpath)
+
+
+def getBasedata(election):
+ "Get base data from an election"
+ elpath = os.path.join(homedir, "issues", election)
+ if os.path.isdir(elpath):
+ with open(elpath + "/basedata.json", "r") as f:
+ data = f.read()
+ f.close()
+ basedata = json.loads(data)
+ if hideHash and 'hash' in basedata:
+ del basedata['hash']
+ basedata['id'] = election
+ return basedata
+ return None
+
+def close(election, reopen = False):
+ "Mark an election as closed"
+
+ elpath = os.path.join(homedir, "issues", election)
+ basedata = getBasedata(election)
+ if reopen:
+ basedata['closed'] = False
+ else:
+ basedata['closed'] = True
+ with open(elpath + "/basedata.json", "w") as f:
+ f.write(json.dumps(basedata))
+ f.close()
+
+
+def getIssue(electionID, issueID):
+ "Get JSON data from an issue"
+ issuedata = None
+ ihash = ""
+ issuepath = os.path.join(homedir, "issues", electionID, issueID) + ".json"
+ if os.path.isfile(issuepath):
+ with open(issuepath, "r") as f:
+ data = f.read()
+ ihash = hashlib.sha224(data).hexdigest()
+ f.close()
+ issuedata = json.loads(data)
+
+ return issuedata, ihash
+
+
+def getVotes(electionID, issueID):
+ "Read votes from the vote file"
+ rvotes = getVotesRaw(electionID, issueID)
+ votes = {}
+ for key in rvotes:
+ votes[key] = rvotes[key]['vote']
+ return {}
+
+
+def getVotesRaw(electionID, issueID):
+ issuepath = os.path.join(homedir, "issues", electionID, issueID) + ".json.votes"
+ if os.path.isfile(issuepath):
+ with open(issuepath, "r") as f:
+ votes = json.loads(f.read())
+ f.close()
+ return votes
+ return {}
+
+
+def createElection(eid, basedata):
+ elpath = os.path.join(homedir, "issues", eid)
+ os.mkdir(elpath)
+ with open(elpath + "/basedata.json", "w") as f:
+ f.write(json.dumps(basedata))
+ f.close()
+ with open(elpath + "/voters.json", "w") as f:
+ f.write("{}")
+ f.close()
+
+
+def updateElection(electionID, basedata):
+ elpath = os.path.join(homedir, "issues", electionID)
+ with open(elpath + "/basedata.json", "w") as f:
+ f.write(json.dumps(basedata))
+ f.close()
+
+def updateIssue(electionID, issueID, issueData):
+ issuepath = os.path.join(homedir, "issues", electionID, issueID) + ".json"
+ with open(issuepath, "w") as f:
+ f.write(json.dumps(issueData))
+ f.close()
+
+
+def listIssues(election):
+ "List all issues in an election"
+ issues = []
+ elpath = os.path.join(homedir, "issues", election)
+ if os.path.isdir(elpath):
+ issues = [f.strip(".json") for f in os.listdir(elpath) if os.path.isfile(os.path.join(elpath, f)) and f != "basedata.json" and f != "voters.json" and f.endswith(".json")]
+ return issues
+
+def listElections():
+ "List all elections"
+ elections = []
+ path = os.path.join(homedir, "issues")
+ if os.path.isdir(path):
+ elections = [f for f in os.listdir(path) if os.path.isdir(os.path.join(path, f))]
+ return elections
+
+def vote(electionID, issueID, uid, vote):
+ "Casts a vote on an issue"
+ votes = {}
+ issuepath = os.path.join(homedir, "issues", electionID, issueID) + ".json"
+ if os.path.isfile(issuepath + ".votes"):
+ with open(issuepath + ".votes", "r") as f:
+ votes = json.loads(f.read())
+ f.close()
+ votes[uid] = {
+ 'vote': vote,
+ 'timestamp': time.time()
+ }
+ with open(issuepath + ".votes", "w") as f:
+ f.write(json.dumps(votes))
+ f.close()
+
+
+def deleteIssue(electionID, issueID):
+ "Deletes an issue if it exists"
+
+ issuepath = os.path.join(homedir, "issues", electionID, issueID) + ".json"
+ if os.path.isfile(issuepath):
+ os.unlink(issuepath)
+ if os.path.isfile(issuepath + ".votes"):
+ os.unlink(issuepath + ".votes")
+
+def createIssue(electionID, issueID, data):
+ issuepath = os.path.join(homedir, "issues", electionID, issueID) + ".json"
+ with open(issuepath, "w") as f:
+ f.write(json.dumps(data))
+ f.close()
+
+def voter_get(electionID, votekey):
+ "Get vote UID/email with a given vote key hash"
+ elpath = os.path.join(homedir, "issues", electionID)
+ with open(elpath + "/voters.json", "r") as f:
+ voters = json.loads(f.read())
+ f.close()
+ for voter in voters:
+ if voters[voter] == xhash:
+ return voter
+ return None
+
+def voter_add(election, PID, xhash):
+ elpath = os.path.join(homedir, "issues", election)
+ with open(elpath + "/voters.json", "r") as f:
+ voters = json.loads(f.read())
+ f.close()
+ voters[PID] = xhash
+ with open(elpath + "/voters.json", "w") as f:
+ f.write(json.dumps(voters))
+ f.close()
+
+def voter_remove(election, UID):
+ elpath = os.path.join(homedir, "issues", election)
+ with open(elpath + "/voters.json", "r") as f:
+ voters = json.loads(f.read())
+ f.close()
+ if UID in voters:
+ del voters[UID]
+ with open(elpath + "/voters.json", "w") as f:
+ f.write(json.dumps(voters))
+ f.close()
+
+def voter_has_voted(election, issue, uid):
+ path = os.path.join(homedir, "issues", election, issue)
+ votes = {}
+ if os.path.isfile(path + ".json.votes"):
+ with open(path + ".json.votes", "r") as f:
+ votes = json.loads(f.read())
+ f.close()
+ return True if uid in votes else False
+
+constants.appendBackend( {
+ 'id': 'file',
+ 'init': init,
+ 'document_exists': exists,
+ 'get_basedata': getBasedata,
+ 'election_close': close,
+ 'election_vote': vote,
+ 'election_list': listElections,
+ 'issue_list': listIssues,
+ 'election_create': createElection,
+ 'issue_create': createIssue,
+ 'issue_delete': deleteIssue,
+ 'election_update': updateElection,
+ 'issue_update': updateIssue,
+ 'issue_get': getIssue,
+ 'vote': vote,
+ 'votes_get': getVotes,
+ 'votes_get_raw': getVotesRaw,
+ 'voter_get_uid': voter_get,
+ 'voter_add': voter_add,
+ 'voter_remove': voter_remove,
+ 'voter_has_voted': voter_has_voted
+})
\ No newline at end of file
Re: svn commit: r1669902 - in /steve/trunk/pysteve/lib/backends: ./ __init__.py es.py files.py
Posted by "Alan D. Cabrera" <li...@toolazydogs.com>.
> On Mar 29, 2015, at 6:19 AM, Daniel Gruno <hu...@apache.org> wrote:
>
>
>
> On 2015-03-29 15:10, Greg Stein wrote:
>> On Sun, Mar 29, 2015 at 7:38 AM, <hu...@apache.org> wrote:
>>> ...
>>> +++ steve/trunk/pysteve/lib/backends/es.py Sun Mar 29 12:38:08 2015
>>>
>>> ...
>>> +constants.appendBackend( {
>>> + 'id': 'elasticsearch',
>>> + 'init': init,
>>> + 'document_exists': exists,
>>> + 'get_basedata': getBasedata,
>>> + 'election_close': close,
>>> + 'election_vote': vote,
>>> + 'election_list': listElections,
>>> + 'issue_list': listIssues,
>>> + 'election_create': createElection,
>>> + 'issue_create': createIssue,
>>> + 'issue_delete': deleteIssue,
>>> + 'election_update': updateElection,
>>> + 'issue_update': updateIssue,
>>> + 'issue_get': getIssue,
>>> + 'vote': vote,
>>> + 'votes_get': getVotes,
>>> + 'votes_get_raw': getVotesRaw,
>>> + 'voter_get_uid': voter_get,
>>> + 'voter_add': voter_add,
>>> + 'voter_remove': voter_remove,
>>> + 'voter_has_voted': voter_has_voted
>>> +})
>>>
>> Why a dictionary mapping names to functions? .. that is why we have classes.
> Yep, a class would be better suited, same thing for the vote types I can imagine, and is on my TODO list (I had a bit of a brain fart while porting it, so I just copied over the structure from vote types).
> I am currently pressed for time (heading out the door), so I won't be able to get this done till later this evening, hence getting in a quicker version first. I wanted something down on paper before I left, so Chris could get started whenever he wants on a Sold backend, but yes, I do realize it's not ideal nor a final piece of work.
Take a look at my last checkin. I think this is what we want in terms of commands and plugins.
Regards,
Alan
Re: svn commit: r1669902 - in /steve/trunk/pysteve/lib/backends:
./ __init__.py es.py files.py
Posted by Daniel Gruno <hu...@apache.org>.
On 2015-03-29 15:10, Greg Stein wrote:
> On Sun, Mar 29, 2015 at 7:38 AM, <hu...@apache.org> wrote:
>> ...
>> +++ steve/trunk/pysteve/lib/backends/es.py Sun Mar 29 12:38:08 2015
>>
>> ...
>> +constants.appendBackend( {
>> + 'id': 'elasticsearch',
>> + 'init': init,
>> + 'document_exists': exists,
>> + 'get_basedata': getBasedata,
>> + 'election_close': close,
>> + 'election_vote': vote,
>> + 'election_list': listElections,
>> + 'issue_list': listIssues,
>> + 'election_create': createElection,
>> + 'issue_create': createIssue,
>> + 'issue_delete': deleteIssue,
>> + 'election_update': updateElection,
>> + 'issue_update': updateIssue,
>> + 'issue_get': getIssue,
>> + 'vote': vote,
>> + 'votes_get': getVotes,
>> + 'votes_get_raw': getVotesRaw,
>> + 'voter_get_uid': voter_get,
>> + 'voter_add': voter_add,
>> + 'voter_remove': voter_remove,
>> + 'voter_has_voted': voter_has_voted
>> +})
>>
> Why a dictionary mapping names to functions? .. that is why we have classes.
Yep, a class would be better suited, same thing for the vote types I can
imagine, and is on my TODO list (I had a bit of a brain fart while
porting it, so I just copied over the structure from vote types).
I am currently pressed for time (heading out the door), so I won't be
able to get this done till later this evening, hence getting in a
quicker version first. I wanted something down on paper before I left,
so Chris could get started whenever he wants on a Sold backend, but yes,
I do realize it's not ideal nor a final piece of work.
With regards,
Daniel.
>
> -g
>
Re: svn commit: r1669902 - in /steve/trunk/pysteve/lib/backends: ./
__init__.py es.py files.py
Posted by Greg Stein <gs...@gmail.com>.
On Sun, Mar 29, 2015 at 7:38 AM, <hu...@apache.org> wrote:
>...
> +++ steve/trunk/pysteve/lib/backends/es.py Sun Mar 29 12:38:08 2015
>
>...
> +constants.appendBackend( {
> + 'id': 'elasticsearch',
> + 'init': init,
> + 'document_exists': exists,
> + 'get_basedata': getBasedata,
> + 'election_close': close,
> + 'election_vote': vote,
> + 'election_list': listElections,
> + 'issue_list': listIssues,
> + 'election_create': createElection,
> + 'issue_create': createIssue,
> + 'issue_delete': deleteIssue,
> + 'election_update': updateElection,
> + 'issue_update': updateIssue,
> + 'issue_get': getIssue,
> + 'vote': vote,
> + 'votes_get': getVotes,
> + 'votes_get_raw': getVotesRaw,
> + 'voter_get_uid': voter_get,
> + 'voter_add': voter_add,
> + 'voter_remove': voter_remove,
> + 'voter_has_voted': voter_has_voted
> +})
>
Why a dictionary mapping names to functions? .. that is why we have classes.
-g