You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2013/09/24 19:07:51 UTC
[12/50] git commit: [#6535] ticket:417 basic github tickets import
[#6535] ticket:417 basic github tickets import
Conflicts:
ForgeImporters/forgeimporters/base.py
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/5a82f13a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/5a82f13a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/5a82f13a
Branch: refs/heads/cj/6422
Commit: 5a82f13a32c497cf6d858a3c8155c5e274d34fc6
Parents: 8a32af8
Author: Anton Kasyanov <mi...@gmail.com>
Authored: Mon Aug 26 18:56:42 2013 +0300
Committer: Dave Brondsema <db...@slashdotmedia.com>
Committed: Thu Sep 19 14:46:49 2013 +0000
----------------------------------------------------------------------
ForgeImporters/forgeimporters/base.py | 1 -
.../forgeimporters/github/__init__.py | 25 ++++-
ForgeImporters/forgeimporters/github/tracker.py | 112 ++++++++++++++++++-
3 files changed, 135 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5a82f13a/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index 2f296a0..ea01968 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -210,7 +210,6 @@ class ProjectImporter(BaseController):
tools = {}
for ep in h.iter_entry_points('allura.importers'):
epv = ep.load()
- print epv, epv.source, self.source
if epv.source == self.source:
tools[ep.name] = epv()
return tools
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5a82f13a/ForgeImporters/forgeimporters/github/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/__init__.py b/ForgeImporters/forgeimporters/github/__init__.py
index 5cd6dc1..35c5d03 100644
--- a/ForgeImporters/forgeimporters/github/__init__.py
+++ b/ForgeImporters/forgeimporters/github/__init__.py
@@ -25,12 +25,22 @@ log = logging.getLogger(__name__)
class GitHubProjectExtractor(base.ProjectExtractor):
PAGE_MAP = {
- 'project_info': 'https://api.github.com/repos/{project_name}',
+ 'project_info': 'https://api.github.com/repos/{project}',
+ 'issues': 'https://api.github.com/repos/{project}/issues',
}
+ POSSIBLE_STATES = ('opened', 'closed')
def parse_page(self, page):
return json.loads(page.read().decode('utf8'))
+ def __init__(self, allura_project, gh_project_name, page):
+ self.project = allura_project
+ self.gh_project_name = gh_project_name
+ self.url = self.PAGE_MAP[page].format(
+ project=urllib.quote(gh_project_name),
+ )
+ self.page = json.loads(urllib2.urlopen(self.url).read().decode('utf8'))
+
def get_summary(self):
return self.get_page('project_info').get('description')
@@ -39,3 +49,16 @@ class GitHubProjectExtractor(base.ProjectExtractor):
def get_repo_url(self):
return self.get_page('project_info').get('clone_url')
+
+ def iter_issues(self):
+ # github api doesn't allow getting closed and opened tickets in one query
+ issues = []
+ self.url += '?state={state}'
+ for state in self.POSSIBLE_STATES:
+ issue_list_url = self.url.format(
+ state=state,
+ )
+ issues += json.loads(urllib2.urlopen(issue_list_url).read().decode('utf8'))
+ issues.sort(key=lambda x: x['number'])
+ for issue in issues:
+ yield (issue['number'], issue)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5a82f13a/ForgeImporters/forgeimporters/github/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tracker.py b/ForgeImporters/forgeimporters/github/tracker.py
index 798da5c..4aa73a4 100644
--- a/ForgeImporters/forgeimporters/github/tracker.py
+++ b/ForgeImporters/forgeimporters/github/tracker.py
@@ -1,8 +1,118 @@
+from datetime import datetime
+
+from allura import model as M
+from allura.lib import helpers as h
+from ming.orm import session, ThreadLocalORMSession
+from pylons import tmpl_context as c
+from pylons import app_globals as g
+
+from . import GitHubProjectExtractor
from ..base import ToolImporter
from forgetracker.tracker_main import ForgeTrackerApp
+from forgetracker import model as TM
+
+
class GitHubTrackerImporter(ToolImporter):
source = 'GitHub'
target_app = ForgeTrackerApp
controller = None
- tool_label = 'Issues'
\ No newline at end of file
+ tool_label = 'Issues'
+ max_ticket_num = 0
+
+ def import_tool(self, project, user, project_name, mount_point=None,
+ mount_label=None, **kw):
+ app = project.install_app('tickets', mount_point, mount_label,
+ EnableVoting=True,
+ open_status_names='New Accepted',
+ closed_status_names='Done',
+ )
+ ThreadLocalORMSession.flush_all()
+ extractor = GitHubProjectExtractor(
+ project,
+ '{}/{}'.format(kw['user_name'],project_name),
+ 'issues',
+ )
+ try:
+ M.session.artifact_orm_session._get().skip_mod_date = True
+ with h.push_config(c, user=M.User.anonymous(), app=app):
+ for ticket_num, issue in extractor.iter_issues():
+ self.max_ticket_num = max(ticket_num, self.max_ticket_num)
+ ticket = TM.Ticket(
+ app_config_id=app.config._id,
+ custom_fields=dict(),
+ ticket_num=ticket_num)
+ self.process_fields(ticket, issue)
+ #self.process_labels(ticket, issue)
+ #self.process_comments(ticket, issue)
+ session(ticket).flush(ticket)
+ session(ticket).expunge(ticket)
+ #app.globals.custom_fields = self.postprocess_custom_fields()
+ app.globals.last_ticket_num = self.max_ticket_num
+ ThreadLocalORMSession.flush_all()
+ g.post_event('project_updated')
+ app.globals.invalidate_bin_counts()
+ return app
+ finally:
+ M.session.artifact_orm_session._get().skip_mod_date = False
+
+ def process_fields(self, ticket, issue):
+ ticket.summary = issue['title']
+ ticket.status = issue['state']
+ ticket.created_date = datetime.strptime(issue['created_at'], '%Y-%m-%dT%H:%M:%SZ')
+ ticket.mod_date = datetime.strptime(issue['updated_at'], '%Y-%m-%dT%H:%M:%SZ')
+ if issue['assignee']:
+ owner_line = '*Originally owned by:* {}\n'.format(issue['assignee']['login'])
+ else:
+ owner_line = ''
+ ticket.description = (
+ u'*Originally created by:* {creator}\n'
+ u'{owner}'
+ u'\n'
+ u'{body}').format(
+ creator=issue['user']['login'],
+ owner=owner_line,
+ body=issue['body'],
+ )
+
+ def process_labels(self, ticket, issue):
+ labels = set()
+ custom_fields = defaultdict(set)
+ for label in issue.get_issue_labels():
+ if u'-' in label:
+ name, value = label.split(u'-', 1)
+ cf = self.custom_field(name)
+ cf['options'].add(value)
+ custom_fields[cf['name']].add(value)
+ if cf['name'] == '_milestone' and ticket.status in c.app.globals.open_status_names:
+ self.open_milestones.add(value)
+ else:
+ labels.add(label)
+ ticket.labels = list(labels)
+ ticket.custom_fields = {n: u', '.join(sorted(v)) for n,v in custom_fields.iteritems()}
+
+ def process_comments(self, ticket, issue):
+ for comment in issue.iter_comments():
+ p = ticket.discussion_thread.add_post(
+ text = comment.annotated_text,
+ ignore_security = True,
+ timestamp = datetime.strptime(comment.created_date, '%c'),
+ )
+ p.add_multiple_attachments(comment.attachments)
+
+ def postprocess_custom_fields(self):
+ custom_fields = []
+ for name, field in self.custom_fields.iteritems():
+ if field['name'] == '_milestone':
+ field['milestones'] = [{
+ 'name': milestone,
+ 'due_date': None,
+ 'complete': milestone not in self.open_milestones,
+ } for milestone in sorted(field['options'])]
+ field['options'] = ''
+ elif field['type'] == 'select':
+ field['options'] = ' '.join(field['options'])
+ else:
+ field['options'] = ''
+ custom_fields.append(field)
+ return custom_fields
\ No newline at end of file