You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by jo...@apache.org on 2014/01/10 19:39:50 UTC
[1/4] git commit: [#6484] ticket:492 Fix tests failing due to
tracwikiimporter is not available
Updated Branches:
refs/heads/cj/6484 [created] f412cebcf
[#6484] ticket:492 Fix tests failing due to tracwikiimporter is not available
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/f412cebc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/f412cebc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/f412cebc
Branch: refs/heads/cj/6484
Commit: f412cebcfeb5308690840e3adb5677fdc265c4ce
Parents: c107eb6
Author: Igor Bondarenko <je...@gmail.com>
Authored: Thu Jan 2 12:32:03 2014 +0200
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Fri Jan 10 18:02:22 2014 +0000
----------------------------------------------------------------------
.../forgeimporters/trac/tests/test_tickets.py | 109 +++++++++++--------
ForgeImporters/forgeimporters/trac/tickets.py | 5 +-
2 files changed, 67 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/f412cebc/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index 8cb687e..a822e97 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -17,7 +17,7 @@
import json
import os
-from datetime import datetime
+import sys
from unittest import TestCase
from mock import Mock, patch, MagicMock
@@ -39,13 +39,33 @@ from forgeimporters.trac.tickets import (
)
+
class TestTracTicketImporter(TestCase):
+
+ def mock_tracwikiimporter(self):
+ if module_not_available('tracwikiimporter'):
+ tracwikiimporter_mock = MagicMock(name='tracwikiimporter')
+ tracwikiimporter_mock.scripts = MagicMock(name='tracwikiimporter.scripts')
+ tracwikiimporter_mock.scripts.trac_export = MagicMock(name='tracwikiimporter.scripts.trac_export')
+ tracwikiimporter_mock.scripts.trac_export.DateJSONEncoder.return_value.encode = json.dumps
+ sys.modules['tracwikiimporter'] = tracwikiimporter_mock
+ sys.modules['tracwikiimporter.scripts'] = tracwikiimporter_mock.scripts
+ sys.modules['tracwikiimporter.scripts.trac_export'] = tracwikiimporter_mock.scripts.trac_export
+ self.tracwikiimporter_mocked = True
+
+ def tearDown(self):
+ if getattr(self, 'tracwikiimporter_mocked', False):
+ sys.modules.pop('tracwikiimporter', None)
+ sys.modules.pop('tracwikiimporter', None)
+ sys.modules.pop('tracwikiimporter.scripts.trac_export', None)
+ self.tracwikiimporter_mocked = False
+
@patch('forgeimporters.trac.tickets.session')
@patch('forgeimporters.trac.tickets.g')
@patch('forgeimporters.trac.tickets.AuditLog')
@patch('forgeimporters.trac.tickets.TracImportSupport')
- @patch('forgeimporters.trac.tickets.export')
- def test_import_tool(self, export, ImportSupport, AuditLog, g, session):
+ @patch('forgeimporters.trac.tickets.c')
+ def test_import_tool(self, c, ImportSupport, AuditLog, g, session):
user_map = {"orig_user":"new_user"}
importer = TracTicketImporter()
app = Mock(name='ForgeTrackerApp')
@@ -55,54 +75,57 @@ class TestTracTicketImporter(TestCase):
project = Mock(name='Project', shortname='myproject')
project.install_app.return_value = app
user = Mock(name='User', _id='id')
- export.return_value = []
- res = importer.import_tool(project, user,
- mount_point='bugs',
- mount_label='Bugs',
- trac_url='http://example.com/trac/url',
- user_map=json.dumps(user_map),
- )
- self.assertEqual(res, app)
- project.install_app.assert_called_once_with(
- 'Tickets', mount_point='bugs', mount_label='Bugs',
- open_status_names='new assigned accepted reopened',
- closed_status_names='closed',
- import_id={
- 'source': 'Trac',
- 'trac_url': 'http://example.com/trac/url/',
- })
- export.assert_called_once_with('http://example.com/trac/url/')
- ImportSupport.return_value.perform_import.assert_called_once_with(
- json.dumps(export.return_value),
- json.dumps({
- "user_map": user_map,
- "usernames_match": False,
- }),
- )
- AuditLog.log.assert_called_once_with(
- 'import tool bugs from http://example.com/trac/url/',
- project=project, user=user, url='foo')
- g.post_event.assert_called_once_with('project_updated')
+ self.mock_tracwikiimporter()
+ with patch('tracwikiimporter.scripts.trac_export.export', create=True) as export:
+ export.return_value = []
+ res = importer.import_tool(project, user,
+ mount_point='bugs',
+ mount_label='Bugs',
+ trac_url='http://example.com/trac/url',
+ user_map=json.dumps(user_map),
+ )
+ self.assertEqual(res, app)
+ project.install_app.assert_called_once_with(
+ 'Tickets', mount_point='bugs', mount_label='Bugs',
+ open_status_names='new assigned accepted reopened',
+ closed_status_names='closed',
+ import_id={
+ 'source': 'Trac',
+ 'trac_url': 'http://example.com/trac/url/',
+ })
+ export.assert_called_once_with('http://example.com/trac/url/')
+ ImportSupport.return_value.perform_import.assert_called_once_with(
+ json.dumps(export.return_value),
+ json.dumps({
+ "user_map": user_map,
+ "usernames_match": False,
+ }),
+ )
+ AuditLog.log.assert_called_once_with(
+ 'import tool bugs from http://example.com/trac/url/',
+ project=project, user=user, url='foo')
+ g.post_event.assert_called_once_with('project_updated')
@patch('forgeimporters.trac.tickets.session')
@patch('forgeimporters.trac.tickets.h')
- @patch('forgeimporters.trac.tickets.export')
- def test_import_tool_failure(self, export, h, session):
+ def test_import_tool_failure(self, h, session):
importer = TracTicketImporter()
app = Mock(name='ForgeTrackerApp')
project = Mock(name='Project', shortname='myproject')
project.install_app.return_value = app
user = Mock(name='User', _id='id')
- export.side_effect = ValueError
-
- self.assertRaises(ValueError, importer.import_tool, project, user,
- mount_point='bugs',
- mount_label='Bugs',
- trac_url='http://example.com/trac/url',
- user_map=None,
- )
-
- h.make_app_admin_only.assert_called_once_with(app)
+ self.mock_tracwikiimporter()
+ with patch('tracwikiimporter.scripts.trac_export.export', create=True) as export:
+ export.side_effect = ValueError
+
+ self.assertRaises(ValueError, importer.import_tool, project, user,
+ mount_point='bugs',
+ mount_label='Bugs',
+ trac_url='http://example.com/trac/url',
+ user_map=None,
+ )
+
+ h.make_app_admin_only.assert_called_once_with(app)
class TestTracTicketImportController(TestController, TestCase):
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/f412cebc/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index ea30fbd..15341f1 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -39,10 +39,6 @@ from allura.lib.decorators import require_post
from allura.lib import validators as v
from allura.lib import helpers as h
from allura.model import AuditLog
-from tracwikiimporter.scripts.trac_export import (
- export,
- DateJSONEncoder,
- )
from forgeimporters.base import (
ToolImporter,
@@ -119,6 +115,7 @@ class TracTicketImporter(ToolImporter):
session(app.config).flush(app.config)
session(app.globals).flush(app.globals)
try:
+ from tracwikiimporter.scripts.trac_export import export, DateJSONEncoder
with h.push_config(c, app=app):
TracImportSupport().perform_import(
json.dumps(export(trac_url), cls=DateJSONEncoder),
[3/4] git commit: [#6484] ticket:492 Move trac_export script to
tracwikiimporter
Posted by jo...@apache.org.
[#6484] ticket:492 Move trac_export script to tracwikiimporter
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/c107eb61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/c107eb61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/c107eb61
Branch: refs/heads/cj/6484
Commit: c107eb610207ad69b01a416c856af60b620862b3
Parents: 5cd4382
Author: Igor Bondarenko <je...@gmail.com>
Authored: Thu Jan 2 11:03:56 2014 +0200
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Fri Jan 10 18:02:22 2014 +0000
----------------------------------------------------------------------
Allura/allura/scripts/trac_export.py | 313 -------------------
.../forgeimporters/trac/tests/test_tickets.py | 5 +-
ForgeImporters/forgeimporters/trac/tickets.py | 2 +-
scripts/trac_export.py | 2 +-
4 files changed, 5 insertions(+), 317 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c107eb61/Allura/allura/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/trac_export.py b/Allura/allura/scripts/trac_export.py
deleted file mode 100644
index 8be9fe0..0000000
--- a/Allura/allura/scripts/trac_export.py
+++ /dev/null
@@ -1,313 +0,0 @@
-#!/usr/bin/env python
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import logging
-import sys
-import csv
-import urlparse
-import urllib2
-import json
-import time
-import re
-from optparse import OptionParser
-from itertools import islice
-
-import feedparser
-from BeautifulSoup import BeautifulSoup, NavigableString
-import dateutil.parser
-import pytz
-
-try:
- from forgeimporters.base import ProjectExtractor
- urlopen = ProjectExtractor.urlopen
-except ImportError:
- try:
- from allura.lib.helpers import urlopen
- except ImportError:
- from urllib2 import urlopen
-
-log = logging.getLogger(__name__)
-
-
-def parse_options():
- optparser = OptionParser(usage=''' %prog <Trac URL>
-
-Export ticket data from a Trac instance''')
- optparser.add_option('-o', '--out-file', dest='out_filename', help='Write to file (default stdout)')
- optparser.add_option('--no-attachments', dest='do_attachments', action='store_false', default=True, help='Export attachment info')
- optparser.add_option('--only-tickets', dest='only_tickets', action='store_true', help='Export only ticket list')
- optparser.add_option('--start', dest='start_id', type='int', default=1, help='Start with given ticket numer (or next accessible)')
- optparser.add_option('--limit', dest='limit', type='int', default=None, help='Limit number of tickets')
- optparser.add_option('-v', '--verbose', dest='verbose', action='store_true', help='Verbose operation')
- options, args = optparser.parse_args()
- if len(args) != 1:
- optparser.error("Wrong number of arguments.")
- return options, args
-
-
-class TracExport(object):
-
- PAGE_SIZE = 100
- TICKET_URL = 'ticket/%d'
- QUERY_MAX_ID_URL = 'query?col=id&order=id&desc=1&max=2'
- QUERY_BY_PAGE_URL = 'query?col=id&col=time&col=changetime&order=id&max=' + str(PAGE_SIZE)+ '&page=%d'
- ATTACHMENT_LIST_URL = 'attachment/ticket/%d/'
- ATTACHMENT_URL = 'raw-attachment/ticket/%d/%s'
-
- FIELD_MAP = {
- 'reporter': 'submitter',
- 'owner': 'assigned_to',
- }
-
- def __init__(self, base_url, start_id=1, verbose=False, do_attachments=True):
- """start_id - start with at least that ticket number (actual returned
- ticket may have higher id if we don't have access to exact
- one).
- """
- self.base_url = base_url.rstrip('/') + '/'
- # Contains additional info for a ticket which cannot
- # be get with single-ticket export (create/mod times is
- # and example).
- self.ticket_map = {}
- self.start_id = start_id
- self.page = (start_id - 1) / self.PAGE_SIZE + 1
- self.verbose = verbose
- self.do_attachments = do_attachments
- self.exhausted = False
- self.ticket_queue = self.next_ticket_ids()
-
- def remap_fields(self, dict):
- "Remap fields to adhere to standard taxonomy."
- out = {}
- for k, v in dict.iteritems():
- out[self.FIELD_MAP.get(k, k)] = v
-
- out['id'] = int(out['id'])
- if 'private' in out:
- out['private'] = bool(int(out['private']))
- return out
-
- def full_url(self, suburl, type=None):
- url = urlparse.urljoin(self.base_url, suburl)
- if type is None:
- return url
- glue = '&' if '?' in suburl else '?'
- return url + glue + 'format=' + type
-
- def log_url(self, url):
- log.info(url)
- if self.verbose:
- print >>sys.stderr, url
-
- @classmethod
- def trac2z_date(cls, s):
- d = dateutil.parser.parse(s)
- d = d.astimezone(pytz.UTC)
- return d.strftime("%Y-%m-%dT%H:%M:%SZ")
-
- @staticmethod
- def match_pattern(regexp, string):
- m = re.match(regexp, string)
- assert m
- return m.group(1)
-
- def csvopen(self, url):
- self.log_url(url)
- f = urlopen(url)
- # Trac doesn't throw 403 error, just shows normal 200 HTML page
- # telling that access denied. So, we'll emulate 403 ourselves.
- # TODO: currently, any non-csv result treated as 403.
- if not f.info()['Content-Type'].startswith('text/csv'):
- raise urllib2.HTTPError(url, 403, 'Forbidden - emulated', f.info(), f)
- return f
-
- def parse_ticket(self, id):
- # Use CSV export to get ticket fields
- url = self.full_url(self.TICKET_URL % id, 'csv')
- f = self.csvopen(url)
- reader = csv.DictReader(f)
- ticket_fields = reader.next()
- ticket_fields['class'] = 'ARTIFACT'
- ticket = self.remap_fields(ticket_fields)
-
- # Use HTML export to get ticket description and comments
- import html2text
- html2text.BODY_WIDTH = 0
- url = self.full_url(self.TICKET_URL % id)
- self.log_url(url)
- d = BeautifulSoup(urlopen(url))
- self.clean_missing_wiki_links(d)
- desc = d.find('div', 'description').find('div', 'searchable')
- ticket['description'] = html2text.html2text(desc.renderContents('utf8').decode('utf8')) if desc else ''
- comments = []
- for comment in d.findAll('form', action='#comment'):
- c = {}
- c['submitter'] = re.sub(r'.* by ', '', comment.find('h3', 'change').text).strip()
- c['date'] = self.trac2z_date(comment.find('a', 'timeline')['title'].replace(' in Timeline', ''))
- changes = unicode(comment.find('ul', 'changes') or '')
- body = comment.find('div', 'comment')
- body = body.renderContents('utf8').decode('utf8') if body else ''
- c['comment'] = html2text.html2text(changes + body)
- c['class'] = 'COMMENT'
- comments.append(c)
- ticket['comments'] = comments
- return ticket
-
- def parse_ticket_attachments(self, id):
- SIZE_PATTERN = r'(\d+) bytes'
- TIMESTAMP_PATTERN = r'(.+) in Timeline'
- # Scrape HTML to get ticket attachments
- url = self.full_url(self.ATTACHMENT_LIST_URL % id)
- self.log_url(url)
- f = urlopen(url)
- soup = BeautifulSoup(f)
- attach = soup.find('div', id='attachments')
- list = []
- while attach:
- attach = attach.findNext('dt')
- if not attach:
- break
- d = {}
- d['filename'] = attach.a['href'].rsplit('/', 1)[1]
- d['url'] = self.full_url(self.ATTACHMENT_URL % (id, d['filename']))
- size_s = attach.span['title']
- d['size'] = int(self.match_pattern(SIZE_PATTERN, size_s))
- timestamp_s = attach.find('a', {'class': 'timeline'})['title']
- d['date'] = self.trac2z_date(self.match_pattern(TIMESTAMP_PATTERN, timestamp_s))
- d['by'] = attach.find(text=re.compile('added by')).nextSibling.renderContents()
- d['description'] = ''
- # Skip whitespace
- while attach.nextSibling and type(attach.nextSibling) is NavigableString:
- attach = attach.nextSibling
- # if there's a description, there will be a <dd> element, other immediately next <dt>
- if attach.nextSibling and attach.nextSibling.name == 'dd':
- desc_el = attach.nextSibling
- if desc_el:
- # TODO: Convert to Allura link syntax as needed
- d['description'] = ''.join(desc_el.findAll(text=True)).strip()
- list.append(d)
- return list
-
- def get_max_ticket_id(self):
- url = self.full_url(self.QUERY_MAX_ID_URL, 'csv')
- f = self.csvopen(url)
- reader = csv.DictReader(f)
- fields = reader.next()
- print fields
- return int(fields['id'])
-
- def get_ticket(self, id, extra={}):
- '''Get ticket with given id
- extra: extra fields to add to ticket (parsed elsewhere)
- '''
- t = self.parse_ticket(id)
- if self.do_attachments:
- atts = self.parse_ticket_attachments(id)
- if atts:
- t['attachments'] = atts
- t.update(extra)
- return t
-
- def next_ticket_ids(self):
- 'Go thru ticket list and collect available ticket ids.'
- # We could just do CSV export, which by default dumps entire list
- # Alas, for many busy servers with long ticket list, it will just
- # time out. So, let's paginate it instead.
- res = []
-
- url = self.full_url(self.QUERY_BY_PAGE_URL % self.page, 'csv')
- try:
- f = self.csvopen(url)
- except urllib2.HTTPError, e:
- if 'emulated' in e.msg:
- body = e.fp.read()
- if 'beyond the number of pages in the query' in body or 'Log in with a SourceForge account' in body:
- raise StopIteration
- raise
- reader = csv.reader(f)
- cols = reader.next()
- for r in reader:
- if r and r[0].isdigit():
- id = int(r[0])
- extra = {'date': self.trac2z_date(r[1]), 'date_updated': self.trac2z_date(r[2])}
- res.append((id, extra))
- self.page += 1
-
- if len(res) < self.PAGE_SIZE:
- self.exhausted = True
-
- return res
-
- def __iter__(self):
- return self
-
- def next(self):
- while True:
- # queue empty, try to fetch more
- if len(self.ticket_queue) == 0 and not self.exhausted:
- self.ticket_queue = self.next_ticket_ids()
- # there aren't any more, we're really done
- if len(self.ticket_queue) == 0:
- raise StopIteration
- id, extra = self.ticket_queue.pop(0)
- if id >= self.start_id:
- break
- return self.get_ticket(id, extra)
-
- def clean_missing_wiki_links(self, doc):
- for link in doc.findAll('a', 'missing wiki'):
- link.string = link.string.rstrip('?')
-
-
-class DateJSONEncoder(json.JSONEncoder):
- def default(self, obj):
- if isinstance(obj, time.struct_time):
- return time.strftime('%Y-%m-%dT%H:%M:%SZ', obj)
- return json.JSONEncoder.default(self, obj)
-
-
-def export(url, start_id=1, verbose=False, do_attachments=True,
- only_tickets=False, limit=None):
- ex = TracExport(url, start_id=start_id,
- verbose=verbose, do_attachments=do_attachments)
-
- doc = [t for t in islice(ex, limit)]
-
- if not only_tickets:
- doc = {
- 'class': 'PROJECT',
- 'trackers': {'default': {'artifacts': doc}}
- }
- return doc
-
-
-def main():
- options, args = parse_options()
- doc = export(args[0], **vars(options))
-
- out_file = sys.stdout
- if options.out_filename:
- out_file = open(options.out_filename, 'w')
- out_file.write(json.dumps(doc, cls=DateJSONEncoder, indent=2, sort_keys=True))
- # It's bad habit not to terminate lines
- out_file.write('\n')
-
-
-if __name__ == '__main__':
- main()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c107eb61/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index eb97946..8cb687e 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -223,15 +223,16 @@ class TestTracImportSupportFunctional(TestRestApiBase, TestCase):
@with_tracker
@skipif(module_not_available('html2text'))
+ @skipif(module_not_available('tracwikiimporter'))
def test_list(self):
- from allura.scripts.trac_export import TracExport, DateJSONEncoder
+ from tracwikiimporter.scripts.trac_export import TracExport, DateJSONEncoder
csv_fp = open(os.path.dirname(__file__) + '/data/test-list.csv')
html_fp = open(os.path.dirname(__file__) + '/data/test-list.html')
with patch.object(TracExport, 'next_ticket_ids', return_value=[(390, {})]):
te = TracExport('url', do_attachments=False)
te.exhausted = True
te.csvopen = lambda s: csv_fp
- with patch('allura.scripts.trac_export.urlopen', return_value=html_fp):
+ with patch('tracwikiimporter.scripts.trac_export.urlopen', return_value=html_fp):
json_data = {
'class': 'PROJECT',
'trackers': {'default': {'artifacts': list(te)}},
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c107eb61/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index 4fd8fdf..ea30fbd 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -39,7 +39,7 @@ from allura.lib.decorators import require_post
from allura.lib import validators as v
from allura.lib import helpers as h
from allura.model import AuditLog
-from allura.scripts.trac_export import (
+from tracwikiimporter.scripts.trac_export import (
export,
DateJSONEncoder,
)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c107eb61/scripts/trac_export.py
----------------------------------------------------------------------
diff --git a/scripts/trac_export.py b/scripts/trac_export.py
index ac90b17..576d99c 100755
--- a/scripts/trac_export.py
+++ b/scripts/trac_export.py
@@ -18,5 +18,5 @@
# under the License.
if __name__ == '__main__':
- from allura.scripts.trac_export import main
+ from tracwikiimporter.scripts.trac_export import main
main()
[4/4] git commit: [#6484] ticket:492 Move mediawiki import script to
separate repo
Posted by jo...@apache.org.
[#6484] ticket:492 Move mediawiki import script to separate repo
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/0496e5ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/0496e5ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/0496e5ab
Branch: refs/heads/cj/6484
Commit: 0496e5abfd7f0a74f8a1458dfa7667b4b7195922
Parents: 37ecc5e
Author: Igor Bondarenko <je...@gmail.com>
Authored: Mon Dec 30 12:47:20 2013 +0200
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Fri Jan 10 18:02:22 2014 +0000
----------------------------------------------------------------------
.../forgewiki/scripts/wiki2markdown/__init__.py | 18 -
.../scripts/wiki2markdown/extractors.py | 188 -----------
.../forgewiki/scripts/wiki2markdown/loaders.py | 199 -----------
.../scripts/wiki2markdown/wiki2markdown.py | 126 -------
ForgeWiki/forgewiki/tests/test_wiki2markdown.py | 329 -------------------
ForgeWiki/setup.py | 3 -
6 files changed, 863 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0496e5ab/ForgeWiki/forgewiki/scripts/wiki2markdown/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki2markdown/__init__.py b/ForgeWiki/forgewiki/scripts/wiki2markdown/__init__.py
deleted file mode 100644
index f60b66d..0000000
--- a/ForgeWiki/forgewiki/scripts/wiki2markdown/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from wiki2markdown import Wiki2Markdown
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0496e5ab/ForgeWiki/forgewiki/scripts/wiki2markdown/extractors.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki2markdown/extractors.py b/ForgeWiki/forgewiki/scripts/wiki2markdown/extractors.py
deleted file mode 100644
index 7815ba9..0000000
--- a/ForgeWiki/forgewiki/scripts/wiki2markdown/extractors.py
+++ /dev/null
@@ -1,188 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import logging
-import os
-import shutil
-import json
-import hashlib
-
-log = logging.getLogger(__name__)
-
-
-class MediawikiExtractor(object):
- """Base class for MediaWiki data provider"""
-
- def __init__(self, options):
- self.options = options
- if os.path.exists(self.options.dump_dir):
- # clear dump_dir before extraction (there may be an old data)
- shutil.rmtree(self.options.dump_dir)
- os.makedirs(self.options.dump_dir)
-
- def extract(self):
- """Extract pages with history, attachments, talk-pages, etc"""
- raise NotImplementedError("subclass must override this")
-
-
-class MySQLExtractor(MediawikiExtractor):
- """Extract MediaWiki data to json.
-
- Use connection to MySQL database as a data source.
- """
-
- def __init__(self, options):
- super(MySQLExtractor, self).__init__(options)
- self._connection = None
- self.db_options = {
- 'host': self.options.host or 'localhost',
- 'user': self.options.user,
- 'passwd': self.options.password,
- 'db': self.options.db_name,
- 'port': self.options.port or 3306
- }
-
- def connection(self):
- try:
- import MySQLdb
- except ImportError:
- raise ImportError('GPL library MySQL-python is required for this operation')
-
- if not self._connection:
- self._connection = MySQLdb.connect(**self.db_options)
- return self._connection
-
- def _save(self, content, *paths):
- """Save json to file in local filesystem"""
- out_file = os.path.join(self.options.dump_dir, *paths)
- if not os.path.exists(os.path.dirname(out_file)):
- os.makedirs(os.path.dirname(out_file))
- with open(out_file, 'w') as out:
- out.write(content.encode('utf-8'))
-
- def _save_attachment(self, filepath, *paths):
- """Save attachment in dump directory.
-
- Copy from mediawiki dump directory to our internal dump directory.
-
- args:
- filepath - path to attachment in mediawiki dump.
- *paths - path to internal dump directory.
- """
- out_dir = os.path.join(self.options.dump_dir, *paths)
- if not os.path.exists(out_dir):
- os.makedirs(out_dir)
- shutil.copy(filepath, out_dir)
-
- def _pages(self):
- """Yield page_data for next wiki page"""
- c = self.connection().cursor()
- c.execute('select page.page_id, page.page_title '
- 'from page where page.page_namespace = 0')
- for row in c:
- _id, title = row
- page_data = {
- 'page_id': _id,
- 'title': title,
- }
- yield page_data
-
- def _history(self, page_id):
- """Yield page_data for next revision of wiki page"""
- c = self.connection().cursor()
- c.execute('select revision.rev_timestamp, text.old_text, '
- 'revision.rev_user_text '
- 'from revision '
- 'left join text on revision.rev_text_id = text.old_id '
- 'where revision.rev_page = %s', page_id)
- for row in c:
- timestamp, text, username = row
- page_data = {
- 'timestamp': timestamp,
- 'text': text or '',
- 'username': username
- }
- yield page_data
-
- def _talk(self, page_title):
- """Return page_data for talk page with `page_title` title"""
- c = self.connection().cursor()
- query_attrs = (page_title, 1) # page_namespace == 1 - talk pages
- c.execute('select text.old_text, revision.rev_timestamp, '
- 'revision.rev_user_text '
- 'from page '
- 'left join revision on revision.rev_id = page.page_latest '
- 'left join text on text.old_id = revision.rev_text_id '
- 'where page.page_title = %s and page.page_namespace = %s '
- 'limit 1', query_attrs)
-
- row = c.fetchone()
- if row:
- text, timestamp, username = row
- return {'text': text, 'timestamp': timestamp, 'username': username}
-
- def _attachments(self, page_id):
- """Yield path to next file attached to wiki page"""
- c = self.connection().cursor()
- c.execute('select il_to from imagelinks '
- 'where il_from = %s' % page_id)
- for row in c:
- name = row[0]
- # mediawiki stores attachmets in subdirectories
- # based on md5-hash of filename
- # so we need to build path to file as follows
- md5 = hashlib.md5(name).hexdigest()
- path = os.path.join(self.options.attachments_dir,
- md5[:1], md5[:2], name)
- if os.path.isfile(path):
- yield path
-
- def extract(self):
- self.extract_pages()
-
- def extract_pages(self):
- log.info('Extracting pages...')
- for page in self._pages():
- self.extract_history(page)
- self.extract_talk(page)
- self.extract_attachments(page)
- log.info('Extracting pages done')
-
- def extract_history(self, page):
- page_id = page['page_id']
- for page_data in self._history(page_id):
- page_data.update(page)
- self._save(json.dumps(page_data), 'pages', str(page_id),
- 'history', str(page_data['timestamp']) + '.json')
- log.info('Extracted history for page %s (%s)', page_id, page['title'])
-
- def extract_talk(self, page):
- page_id = page['page_id']
- talk_page_data = self._talk(page['title'])
- if talk_page_data:
- self._save(json.dumps(talk_page_data), 'pages', str(page_id),
- 'discussion.json')
- log.info('Extracted talk for page %s (%s)', page_id, page['title'])
- else:
- log.info('No talk for page %s (%s)', page_id, page['title'])
-
- def extract_attachments(self, page):
- page_id = page['page_id']
- for filepath in self._attachments(page_id):
- self._save_attachment(filepath, 'pages', str(page_id),
- 'attachments')
- log.info('Extracted attachments for page %s (%s)', page_id, page['title'])
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0496e5ab/ForgeWiki/forgewiki/scripts/wiki2markdown/loaders.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki2markdown/loaders.py b/ForgeWiki/forgewiki/scripts/wiki2markdown/loaders.py
deleted file mode 100644
index 588fa64..0000000
--- a/ForgeWiki/forgewiki/scripts/wiki2markdown/loaders.py
+++ /dev/null
@@ -1,199 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import logging
-import os
-import json
-import datetime
-from pylons import tmpl_context as c
-from ming.orm.ormsession import ThreadLocalORMSession
-
-from allura import model as M
-from forgewiki import model as WM
-from forgewiki.converters import mediawiki2markdown
-from forgewiki.converters import mediawiki_internal_links2markdown
-from allura.lib import helpers as h
-from allura.lib import utils
-from allura.model.session import artifact_orm_session
-
-log = logging.getLogger(__name__)
-
-
-class MediawikiLoader(object):
- """Load MediaWiki data from json to Allura wiki tool"""
- TIMESTAMP_FMT = '%Y%m%d%H%M%S'
-
- def __init__(self, options):
- self.options = options
- self.nbhd = M.Neighborhood.query.get(name=options.nbhd)
- if not self.nbhd:
- raise ValueError("Can't find neighborhood with name %s"
- % options.nbhd)
- self.project = M.Project.query.get(shortname=options.project,
- neighborhood_id=self.nbhd._id)
- if not self.project:
- raise ValueError("Can't find project with shortname %s "
- "and neighborhood_id %s"
- % (options.project, self.nbhd._id))
-
- self.wiki = self.project.app_instance('wiki')
- if not self.wiki:
- raise ValueError("Can't find wiki app in given project")
-
- h.set_context(self.project.shortname, 'wiki', neighborhood=self.nbhd)
-
- def load(self):
- try:
- self.project.notifications_disabled = True
- artifact_orm_session._get().skip_mod_date = True
- self.load_pages()
- ThreadLocalORMSession.flush_all()
- log.info('Loading wiki done')
- finally:
- self.project.notifications_disabled = False
- artifact_orm_session._get().skip_mod_date = False
-
- def _pages(self):
- """Yield path to page dump directory for next wiki page"""
- pages_dir = os.path.join(self.options.dump_dir, 'pages')
- pages = []
- if not os.path.isdir(pages_dir):
- return
- pages = os.listdir(pages_dir)
- for directory in pages:
- dir_path = os.path.join(pages_dir, directory)
- if os.path.isdir(dir_path):
- yield dir_path
-
- def _history(self, page_dir):
- """Yield page_data for next wiki page in edit history"""
- page_dir = os.path.join(page_dir, 'history')
- if not os.path.isdir(page_dir):
- return
- pages = os.listdir(page_dir)
- pages.sort() # ensure that history in right order
- for page in pages:
- fn = os.path.join(page_dir, page)
- try:
- with open(fn, 'r') as pages_file:
- page_data = json.load(pages_file)
- except IOError, e:
- log.error("Can't open file: %s", str(e))
- raise
- except ValueError, e:
- log.error("Can't load data from file %s: %s", fn, str(e))
- raise
- yield page_data
-
- def _talk(self, page_dir):
- """Return talk data from json dump"""
- filename = os.path.join(page_dir, 'discussion.json')
- if not os.path.isfile(filename):
- return
- try:
- with open(filename, 'r') as talk_file:
- talk_data = json.load(talk_file)
- except IOError, e:
- log.error("Can't open file: %s", str(e))
- raise
- except ValueError, e:
- log.error("Can't load data from file %s: %s", filename, str(e))
- raise
- return talk_data
-
- def _attachments(self, page_dir):
- """Yield (filename, full path) to next attachment for given page."""
- attachments_dir = os.path.join(page_dir, 'attachments')
- if not os.path.isdir(attachments_dir):
- return
- attachments = os.listdir(attachments_dir)
- for filename in attachments:
- yield filename, os.path.join(attachments_dir, filename)
-
- def load_pages(self):
- """Load pages with edit history from json to Allura wiki tool"""
- log.info('Loading pages into allura...')
- for page_dir in self._pages():
- for page in self._history(page_dir):
- p = WM.Page.upsert(page['title'])
- p.viewable_by = ['all']
- p.text = mediawiki_internal_links2markdown(
- mediawiki2markdown(page['text']),
- page['title'])
- timestamp = datetime.datetime.strptime(page['timestamp'],
- self.TIMESTAMP_FMT)
- p.mod_date = timestamp
- c.user = (M.User.query.get(username=page['username'].lower())
- or M.User.anonymous())
- ss = p.commit()
- ss.mod_date = ss.timestamp = timestamp
-
- # set home to main page
- if page['title'] == 'Main_Page':
- gl = WM.Globals.query.get(app_config_id=self.wiki.config._id)
- if gl is not None:
- gl.root = page['title']
- log.info('Loaded history of page %s (%s)', page['page_id'], page['title'])
-
- self.load_talk(page_dir, page['title'])
- self.load_attachments(page_dir, page['title'])
-
- def load_talk(self, page_dir, page_title):
- """Load talk for page.
-
- page_dir - path to directory with page dump.
- page_title - page title in Allura Wiki
- """
- talk_data = self._talk(page_dir)
- if not talk_data:
- return
- text = mediawiki2markdown(talk_data['text'])
- page = WM.Page.query.get(app_config_id=self.wiki.config._id,
- title=page_title)
- if not page:
- return
- thread = M.Thread.query.get(ref_id=page.index_id())
- if not thread:
- return
- timestamp = datetime.datetime.strptime(talk_data['timestamp'],
- self.TIMESTAMP_FMT)
- c.user = (M.User.query.get(username=talk_data['username'].lower())
- or M.User.anonymous())
- thread.add_post(
- text=text,
- discussion_id=thread.discussion_id,
- thread_id=thread._id,
- timestamp=timestamp,
- ignore_security=True)
- log.info('Loaded talk for page %s', page_title)
-
- def load_attachments(self, page_dir, page_title):
- """Load attachments for page.
-
- page_dir - path to directory with page dump.
- """
- page = WM.Page.query.get(app_config_id=self.wiki.config._id,
- title=page_title)
- for filename, path in self._attachments(page_dir):
- try:
- with open(path) as fp:
- page.attach(filename, fp,
- content_type=utils.guess_mime_type(filename))
- except IOError, e:
- log.error("Can't open file: %s", str(e))
- raise
- log.info('Loaded attachments for page %s.', page_title)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0496e5ab/ForgeWiki/forgewiki/scripts/wiki2markdown/wiki2markdown.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki2markdown/wiki2markdown.py b/ForgeWiki/forgewiki/scripts/wiki2markdown/wiki2markdown.py
deleted file mode 100644
index 7d5b2b4..0000000
--- a/ForgeWiki/forgewiki/scripts/wiki2markdown/wiki2markdown.py
+++ /dev/null
@@ -1,126 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import argparse
-import logging
-import shutil
-import tempfile
-
-from tg import config
-
-from allura.lib import helpers as h
-from allura.scripts import ScriptTask
-
-from forgewiki.scripts.wiki2markdown.extractors import MySQLExtractor
-from forgewiki.scripts.wiki2markdown.loaders import MediawikiLoader
-
-log = logging.getLogger(__name__)
-
-
-class Wiki2Markdown(ScriptTask):
- """Import MediaWiki to Allura Wiki tool"""
- @classmethod
- def parser(cls):
- parser = argparse.ArgumentParser(description='Import wiki from'
- 'mediawiki-dump to allura wiki')
- parser.add_argument('-e', '--extract-only', action='store_true',
- dest='extract',
- help='Store data from the mediawiki-dump '
- 'on the local filesystem; not load into Allura')
- parser.add_argument('-l', '--load-only', action='store_true', dest='load',
- help='Load into Allura previously-extracted data')
- parser.add_argument('-d', '--dump-dir', dest='dump_dir', default='',
- help='Directory for dump files')
- parser.add_argument('-n', '--neighborhood', dest='nbhd', default='',
- help='Neighborhood name to load data')
- parser.add_argument('-p', '--project', dest='project', default='',
- help='Project shortname to load data into')
- parser.add_argument('-a', '--attachments-dir', dest='attachments_dir',
- help='Path to directory with mediawiki attachments dump',
- default='')
- parser.add_argument('--db_config_prefix', dest='db_config_prefix',
- help='Key prefix (e.g. "legacy.") in ini file to '
- 'use instead of commandline db params')
- parser.add_argument('-s', '--source', dest='source', default='mysql',
- help='Database type to extract from (only mysql for now)')
- parser.add_argument('--db_name', dest='db_name', default='mediawiki',
- help='Database name')
- parser.add_argument('--host', dest='host', default='localhost',
- help='Database host')
- parser.add_argument('--port', dest='port', type=int, default=0,
- help='Database port')
- parser.add_argument('--user', dest='user', default='',
- help='User for database connection')
- parser.add_argument('--password', dest='password', default='',
- help='Password for database connection')
- parser.add_argument('--keep-dumps', action='store_true', dest='keep_dumps',
- help='Leave dump files on disk after run')
- return parser
-
- @classmethod
- def execute(cls, options):
- options = cls.handle_options(options)
-
- try:
- if options.extract:
- MySQLExtractor(options).extract()
- if options.load:
- MediawikiLoader(options).load()
- finally:
- if not options.keep_dumps:
- shutil.rmtree(options.dump_dir)
-
- @classmethod
- def handle_options(cls, options):
- if not options.extract and not options.load:
- # if action doesn't specified - do both
- options.extract = True
- options.load = True
-
- if not options.dump_dir:
- if options.load and not options.extract:
- raise ValueError('You must specify directory containing dump files')
- else:
- options.dump_dir = tempfile.mkdtemp()
- log.info("Writing temp files to %s", options.dump_dir)
-
- if options.load and (not options.project or not options.nbhd):
- raise ValueError('You must specify neighborhood and project '
- 'to load data')
-
- if options.extract:
- if options.db_config_prefix:
- for k, v in h.config_with_prefix(config, options.db_config_prefix).iteritems():
- if k == 'port':
- v = int(v)
- setattr(options, k, v)
-
- if options.source == 'mysql':
- pass
- elif options.source in ('sqlite', 'postgres', 'sql-dump'):
- raise ValueError('This source not implemented yet. Only mysql for now')
- else:
- raise ValueError('You must specify a valid data source')
-
- if not options.attachments_dir:
- raise ValueError('You must specify path to directory with mediawiki attachmets dump.')
-
- return options
-
-
-if __name__ == '__main__':
- Wiki2Markdown.main()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0496e5ab/ForgeWiki/forgewiki/tests/test_wiki2markdown.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/tests/test_wiki2markdown.py b/ForgeWiki/forgewiki/tests/test_wiki2markdown.py
deleted file mode 100644
index ff5c662..0000000
--- a/ForgeWiki/forgewiki/tests/test_wiki2markdown.py
+++ /dev/null
@@ -1,329 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-import json
-from datetime import datetime
-
-import mock
-from IPython.testing.decorators import module_not_available, skipif
-from pylons import app_globals as g
-
-from forgewiki.scripts.wiki2markdown.extractors import MySQLExtractor
-from forgewiki.scripts.wiki2markdown.loaders import MediawikiLoader
-from alluratest.controller import setup_basic_test
-from allura import model as M
-from forgewiki import model as WM
-from allura.lib import helpers as h
-
-from pylons import tmpl_context as context
-
-
-class TestMySQLExtractor(object):
-
- def setUp(self):
- setup_basic_test()
- self.options = mock.Mock()
- self.options.dump_dir = os.path.join(g.tmpdir, 'w2m_test')
-
- # monkey-patch MySQLExtractor for test
- def pages(self):
- yield {'page_id': 1, 'title': 'Test title'}
- yield {'page_id': 2, 'title': 'Main_Page'}
- yield {'page_id': 3, 'title': 'Test'}
-
- def history(self, page_id):
- data = {
- 1: [
- {'timestamp': 1, 'text': "Test", 'username': 'test-user'},
- {'timestamp': 2, 'text': "Test Text", 'username': 'bad'}
- ],
- 2: [
- {'timestamp': 1, 'text': "Main_Page", 'username': 'b'},
- {'timestamp': 2, 'text': "Main_Page text", 'username': 'b'}
- ],
- 3: [
- {'timestamp': 1, 'text': "Some test text", 'username': ''},
- {'timestamp': 2, 'text': "", 'username': ''}
- ]
- }
- revisions = data[page_id]
- for rev in revisions:
- yield rev
-
- def talk(self, page_title):
- return {
- 'text': 'Talk for page %s.' % page_title,
- 'timestamp': 1,
- 'username': 'test-user'
- }
-
- def attachments(self, *args, **kwargs):
- # make 'empty' iterator
- if False:
- yield
-
- MySQLExtractor._pages = pages
- MySQLExtractor._history = history
- MySQLExtractor._talk = talk
- MySQLExtractor._attachments = attachments
- self.extractor = MySQLExtractor(self.options)
-
- def test_extract_pages(self):
- """Test that pages and edit history extracted properly"""
- self.extractor.extract_pages()
-
- # rev 1 of page 1
- with open(os.path.join(self.options.dump_dir, 'pages/1/history/1.json'), 'r') as f:
- page = json.load(f)
- res_page = {
- 'timestamp': 1,
- 'text': 'Test',
- 'page_id': 1,
- 'title': 'Test title',
- 'username': 'test-user'
- }
- assert page == res_page
-
- # rev 2 of page 1
- with open(os.path.join(self.options.dump_dir, 'pages/1/history/2.json'), 'r') as f:
- page = json.load(f)
- res_page = {
- 'timestamp': 2,
- 'text': 'Test Text',
- 'page_id': 1,
- 'title': 'Test title',
- 'username': 'bad'
- }
- assert page == res_page
-
- # rev 1 of page 2
- with open(os.path.join(self.options.dump_dir, 'pages/2/history/1.json'), 'r') as f:
- page = json.load(f)
- res_page = {
- 'timestamp': 1,
- 'text': 'Main_Page',
- 'page_id': 2,
- 'title': 'Main_Page',
- 'username': 'b'
- }
- assert page == res_page
-
- # rev 2 of page 2
- with open(os.path.join(self.options.dump_dir, 'pages/2/history/2.json'), 'r') as f:
- page = json.load(f)
- res_page = {
- 'timestamp': 2,
- 'text': 'Main_Page text',
- 'page_id': 2,
- 'title': 'Main_Page',
- 'username': 'b'
- }
- assert page == res_page
-
- # rev 1 of page 3
- with open(os.path.join(self.options.dump_dir, 'pages/3/history/1.json'), 'r') as f:
- page = json.load(f)
- res_page = {
- 'timestamp': 1,
- 'text': 'Some test text',
- 'page_id': 3,
- 'title': 'Test',
- 'username': ''
- }
- assert page == res_page
-
- # rev 2 of page 3
- with open(os.path.join(self.options.dump_dir, 'pages/3/history/2.json'), 'r') as f:
- page = json.load(f)
- res_page = {
- 'timestamp': 2,
- 'text': '',
- 'page_id': 3,
- 'title': 'Test',
- 'username': ''
- }
- assert page == res_page
-
- def test_extract_talk(self):
- """Test that talk pages extracted properly."""
- pages = [
- {'page_id': 1, 'title': 'Test 1'},
- {'page_id': 2, 'title': 'Test 2'},
- {'page_id': 3, 'title': 'Test 3'},
- ]
- for page in pages:
- self.extractor.extract_talk(page)
-
- with open(os.path.join(self.options.dump_dir, 'pages/1/discussion.json'), 'r') as f:
- page = json.load(f)
- assert page == {
- 'text': 'Talk for page Test 1.',
- 'username': 'test-user',
- 'timestamp': 1}
-
- with open(os.path.join(self.options.dump_dir, 'pages/2/discussion.json'), 'r') as f:
- page = json.load(f)
- assert page == {
- 'text': 'Talk for page Test 2.',
- 'timestamp': 1,
- 'username': 'test-user'}
-
- with open(os.path.join(self.options.dump_dir, 'pages/3/discussion.json'), 'r') as f:
- page = json.load(f)
- assert page == {
- 'text': 'Talk for page Test 3.',
- 'timestamp': 1,
- 'username': 'test-user'}
-
-
-class TestMediawikiLoader(object):
-
- def setUp(self):
- setup_basic_test()
- self.options = mock.Mock()
- # need test project with installed wiki app
- self.options.nbhd = 'Adobe'
- self.options.project = '--init--'
-
- nbhd = M.Neighborhood.query.get(name=self.options.nbhd)
- h.set_context(self.options.project, 'wiki', neighborhood=nbhd)
-
- # monkey-patch MediawikiLoader for test
- def pages(self):
- yield 1
- yield 2
-
- def history(self, page_dir):
- data = {
- 1: [
- {
- 'title': 'Test title',
- 'text': "'''bold''' ''italics''",
- 'page_id': 1,
- 'timestamp': '20120808000001',
- 'username': 'test-user'
- },
- {
- 'title': 'Test title',
- 'text': "'''bold'''",
- 'page_id': 1,
- 'timestamp': '20120809000001',
- 'username': 'test-user'
- },
- ],
- 2: [
- {
- 'title': 'Main',
- 'text': "Main text rev 1",
- 'page_id': 2,
- 'timestamp': '20120808000001',
- 'username': 'bad-user'
- },
- {
- 'title': 'Main',
- 'text': "Main text rev 2",
- 'page_id': 2,
- 'timestamp': '20120809000001',
- 'username': 'bad-user'
- },
-
- ],
- }
- for page in data[page_dir]:
- yield page
-
- def talk(self, page_dir):
- data = {
- 1: {
- 'text': "''Talk page'' for page 1.",
- 'username': 'test-user',
- 'timestamp': '20120809000001'
- },
- 2: {
- 'text': "''Talk page'' for page 2.",
- 'username': 'bad-user',
- 'timestamp': '20120809000001'
- },
- }
- return data[page_dir]
-
- def attachments(self, *args, **kwargs):
- # make 'empty' iterator
- if False:
- yield
-
- MediawikiLoader._pages = pages
- MediawikiLoader._history = history
- MediawikiLoader._talk = talk
- MediawikiLoader._attachments = attachments
- self.loader = MediawikiLoader(self.options)
-
- def get_page(self, title):
- return WM.Page.query.get(app_config_id=context.app.config._id,
- title=title)
-
- def get_post(self, title):
- page = self.get_page(title)
- thread = M.Thread.query.get(ref_id=page.index_id())
- return M.Post.query.get(discussion_id=thread.discussion_id,
- thread_id=thread._id)
-
- @skipif(module_not_available('mediawiki'))
- @mock.patch('allura.model.discuss.g.director')
- def test_load_pages(self, director):
- """Test that pages, edit history and talk loaded properly"""
- self.loader.load_pages()
- page = self.get_page('Test title')
-
- assert page.mod_date == datetime.strptime('20120809000001',
- self.loader.TIMESTAMP_FMT)
- assert page.authors()[0].username == 'test-user'
- assert '**bold**' in page.text
- # _italics should be only in the first revision of page
- assert '_italics_' not in page
-
- page = page.get_version(1)
- assert '**bold** _italics_' in page.text
- assert page.mod_date == datetime.strptime('20120808000001',
- self.loader.TIMESTAMP_FMT)
- assert page.authors()[0].username == 'test-user'
-
- page = self.get_page('Main')
- assert page.mod_date == datetime.strptime('20120809000001',
- self.loader.TIMESTAMP_FMT)
- assert page.authors()[0].username == '*anonymous'
- assert 'Main text rev 2' in page.text
-
- page = page.get_version(1)
- assert page.mod_date == datetime.strptime('20120808000001',
- self.loader.TIMESTAMP_FMT)
- assert page.authors()[0].username == '*anonymous'
- assert 'Main text rev 1' in page.text
-
- # Check that talk pages loaded
- post = self.get_post('Test title')
- assert post.timestamp == datetime.strptime('20120809000001',
- self.loader.TIMESTAMP_FMT)
- assert post.author().username == 'test-user'
- assert '_Talk page_ for page 1.' in post.text
-
- post = self.get_post('Main')
- assert post.timestamp == datetime.strptime('20120809000001',
- self.loader.TIMESTAMP_FMT)
- assert post.author().username == '*anonymous'
- assert '_Talk page_ for page 2.' in post.text
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/0496e5ab/ForgeWiki/setup.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/setup.py b/ForgeWiki/setup.py
index dce7390..4f9f017 100644
--- a/ForgeWiki/setup.py
+++ b/ForgeWiki/setup.py
@@ -42,8 +42,5 @@ setup(name='ForgeWiki',
# -*- Entry points: -*-
[allura]
Wiki=forgewiki.wiki_main:ForgeWikiApp
-
- [paste.paster_command]
- wiki2markdown = forgewiki.command.wiki2markdown:Wiki2MarkDownCommand
""",
)
[2/4] git commit: [#6484] ticket:492 Move wiki_from_trac script to
tracwikiimporter
Posted by jo...@apache.org.
[#6484] ticket:492 Move wiki_from_trac script to tracwikiimporter
Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/5cd4382d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/5cd4382d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/5cd4382d
Branch: refs/heads/cj/6484
Commit: 5cd4382dbf5bb2a1391b9b1cf4c41beacb9e319d
Parents: 0496e5a
Author: Igor Bondarenko <je...@gmail.com>
Authored: Thu Jan 2 10:44:27 2014 +0200
Committer: Cory Johns <cj...@slashdotmedia.com>
Committed: Fri Jan 10 18:02:22 2014 +0000
----------------------------------------------------------------------
ForgeWiki/forgewiki/scripts/__init__.py | 16 --
.../scripts/wiki_from_trac/__init__.py | 18 --
.../scripts/wiki_from_trac/extractors.py | 241 -------------------
.../forgewiki/scripts/wiki_from_trac/loaders.py | 72 ------
.../scripts/wiki_from_trac/wiki_from_trac.py | 69 ------
scripts/allura_import.py | 2 +-
scripts/wiki-export.py | 4 +-
7 files changed, 3 insertions(+), 419 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5cd4382d/ForgeWiki/forgewiki/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/__init__.py b/ForgeWiki/forgewiki/scripts/__init__.py
deleted file mode 100644
index 144e298..0000000
--- a/ForgeWiki/forgewiki/scripts/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5cd4382d/ForgeWiki/forgewiki/scripts/wiki_from_trac/__init__.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki_from_trac/__init__.py b/ForgeWiki/forgewiki/scripts/wiki_from_trac/__init__.py
deleted file mode 100644
index 8d3f8b7..0000000
--- a/ForgeWiki/forgewiki/scripts/wiki_from_trac/__init__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from .wiki_from_trac import WikiFromTrac
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5cd4382d/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py b/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
deleted file mode 100644
index 6415bbf..0000000
--- a/ForgeWiki/forgewiki/scripts/wiki_from_trac/extractors.py
+++ /dev/null
@@ -1,241 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import logging
-import re
-import sys
-import json
-import traceback
-from urllib import quote, unquote
-from urlparse import urljoin, urlsplit
-
-try:
- from forgeimporters.base import ProjectExtractor
- urlopen = ProjectExtractor.urlopen
-except ImportError:
- try:
- from allura.lib.helpers import urlopen
- except ImportError:
- from urllib2 import urlopen
-
-try:
- # Ignore this import if the html2text package is not installed
- import html2text
-except ImportError:
- pass
-
-from BeautifulSoup import BeautifulSoup
-
-log = logging.getLogger(__name__)
-
-
-class WikiExporter(object):
-
- PAGE_LIST_URL = 'wiki/TitleIndex'
- PAGE_URL = 'wiki/%s'
- CONTENT_DIV_ATTRS = {'class': 'wikipage searchable'}
- EXCLUDE_PAGES = [
- 'CamelCase',
- 'InterMapTxt',
- 'InterTrac',
- 'InterWiki',
- 'PageTemplates',
- 'SandBox',
- 'TitleIndex',
- 'TracAccessibility',
- 'TracAdmin',
- 'TracBackup',
- 'TracBrowser',
- 'TracChangeset',
- 'TracEnvironment',
- 'TracFineGrainedPermissions',
- 'TracGuide',
- 'TracImport',
- 'TracIni',
- 'TracInterfaceCustomization',
- 'TracLinks',
- 'TracLogging',
- 'TracNavigation',
- 'TracNotification',
- 'TracPermissions',
- 'TracPlugins',
- 'TracQuery',
- 'TracReports',
- 'TracRevisionLog',
- 'TracRoadmap',
- 'TracRss',
- 'TracSearch',
- 'TracSupport',
- 'TracSyntaxColoring',
- 'TracTickets',
- 'TracTicketsCustomFields',
- 'TracTimeline',
- 'TracUnicode',
- 'TracWiki',
- 'TracWorkflow',
- 'WikiDeletePage',
- 'WikiFormatting',
- 'WikiHtml',
- 'WikiMacros',
- 'WikiNewPage',
- 'WikiPageNames',
- 'WikiProcessors',
- 'WikiRestructuredText',
- 'WikiRestructuredTextLinks',
- 'RecentChanges',
- ]
- RENAME_PAGES = {
- 'WikiStart': 'Home', # Change the start page name to Home
- 'Home': 'WikiStart', # Rename the Home page to WikiStart
- }
-
- def __init__(self, base_url, options):
- self.base_url = base_url
- self.options = options
-
- def export(self, out):
- pages = []
- for title in self.page_list():
- try:
- pages.append(self.get_page(title))
- except:
- self.log('Cannot fetch page %s. Skipping' % title)
- self.log(traceback.format_exc())
- continue
- out.write(json.dumps(pages, indent=2, sort_keys=True))
- out.write('\n')
-
- def log(self, msg):
- log.info(msg)
- if self.options.verbose:
- print >>sys.stderr, msg
-
- def url(self, suburl, type=None):
- url = urljoin(self.base_url, suburl)
- if type is None:
- return url
- glue = '&' if '?' in suburl else '?'
- return url + glue + 'format=' + type
-
- def fetch(self, url):
- return urlopen(url)
-
- def page_list(self):
- url = urljoin(self.base_url, self.PAGE_LIST_URL)
- self.log('Fetching list of pages from %s' % url)
- r = self.fetch(url)
- html = BeautifulSoup(r)
- pages = html.find('div', attrs=self.CONTENT_DIV_ATTRS) \
- .find('ul').findAll('li')
- pages = [page.find('a').text
- for page in pages
- if page.find('a')
- and page.find('a').text not in self.EXCLUDE_PAGES]
- # Remove duplicate entries by converting page list to a set.
- # As we're going to fetch all listed pages,
- # it's safe to destroy the original order of pages.
- return set(pages)
-
- def get_page(self, title):
- title = quote(title)
- convert_method = '_get_page_' + self.options.converter
- content = getattr(self, convert_method)(title)
- page = {
- 'title': self.convert_title(title),
- 'text': self.convert_content(content),
- 'labels': '',
- }
- return page
-
- def _get_page_html2text(self, title):
- url = self.url(self.PAGE_URL % title)
- self.log('Fetching page %s' % url)
- r = self.fetch(url)
- html = BeautifulSoup(r)
- return html.find('div', attrs=self.CONTENT_DIV_ATTRS)
-
- def _get_page_regex(self, title):
- url = self.url(self.PAGE_URL % title, 'txt')
- self.log('Fetching page %s' % url)
- r = self.fetch(url)
- return r
-
- def convert_title(self, title):
- title = self.RENAME_PAGES.get(title, title)
- title = title.replace('/', '-') # Handle subpages
- title = title.rstrip('?') # Links to non-existent pages ends with '?'
- return title
-
- def convert_content(self, content):
- convert_method = '_convert_content_' + self.options.converter
- return getattr(self, convert_method)(content)
-
- def _convert_wiki_toc_to_markdown(self, content):
- """
- Removes contents of div.wiki-toc elements and replaces them with
- the '[TOC]' markdown macro.
- """
- for toc in content('div', attrs={'class': 'wiki-toc'}):
- toc.string = '[TOC]'
- return content
-
- def _convert_content_html2text(self, content):
- html2text.BODY_WIDTH = 0 # Don't wrap lines
- content = self._convert_wiki_toc_to_markdown(content)
- content = html2text.html2text(unicode(content))
- # Convert internal links
- internal_url = urlsplit(self.base_url).path + 'wiki/'
- internal_link_re = r'\[([^]]+)\]\(%s([^)]*)\)' % internal_url
- internal_link = re.compile(internal_link_re, re.UNICODE)
- def sub(match):
- caption = match.group(1)
- page = self.convert_title(match.group(2))
- if caption == page:
- link = '[%s]' % unquote(page)
- else:
- link = '[%s](%s)' % (caption, page)
- return link
- return internal_link.sub(sub, content)
-
- def _convert_content_regex(self, text):
- # https://gist.github.com/sgk/1286682
- text = re.sub('\r\n', '\n', text)
- text = re.sub(r'{{{(.*?)}}}', r'`\1`', text)
-
- def indent4(m):
- return '\n ' + m.group(1).replace('\n', '\n ')
-
- text = re.sub(r'(?sm){{{\n(.*?)\n}}}', indent4, text)
- text = re.sub(r'(?m)^====\s+(.*?)\s+====$', r'#### \1', text)
- text = re.sub(r'(?m)^===\s+(.*?)\s+===$', r'### \1', text)
- text = re.sub(r'(?m)^==\s+(.*?)\s+==$', r'## \1', text)
- text = re.sub(r'(?m)^=\s+(.*?)\s+=$', r'# \1', text)
- text = re.sub(r'^ * ', r'****', text)
- text = re.sub(r'^ * ', r'***', text)
- text = re.sub(r'^ * ', r'**', text)
- text = re.sub(r'^ * ', r'*', text)
- text = re.sub(r'^ \d+. ', r'1.', text)
- a = []
- for line in text.split('\n'):
- if not line.startswith(' '):
- line = re.sub(r'\[(https?://[^\s\[\]]+)\s([^\[\]]+)\]', r'[\2](\1)', line)
- line = re.sub(r'\[(wiki:[^\s\[\]]+)\s([^\[\]]+)\]', r'[\2](/\1/)', line)
- line = re.sub(r'\!(([A-Z][a-z0-9]+){2,})', r'\1', line)
- line = re.sub(r'\'\'\'(.*?)\'\'\'', r'*\1*', line)
- line = re.sub(r'\'\'(.*?)\'\'', r'_\1_', line)
- a.append(line)
- return '\n'.join(a)
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5cd4382d/ForgeWiki/forgewiki/scripts/wiki_from_trac/loaders.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki_from_trac/loaders.py b/ForgeWiki/forgewiki/scripts/wiki_from_trac/loaders.py
deleted file mode 100644
index 55e4480..0000000
--- a/ForgeWiki/forgewiki/scripts/wiki_from_trac/loaders.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import json
-from optparse import OptionParser
-
-from allura.lib.import_api import AlluraImportApiClient
-
-
-def load_data(doc_file_name=None, optparser=None, options=None):
- import_options = {}
- for s in options.import_opts:
- k, v = s.split('=', 1)
- if v == 'false':
- v = False
- import_options[k] = v
-
- user_map = {}
- if options.user_map_file:
- f = open(options.user_map_file)
- try:
- user_map = json.load(f)
- if type(user_map) is not type({}):
- raise ValueError
- for k, v in user_map.iteritems():
- print k, v
- if not isinstance(k, basestring) or not isinstance(v, basestring):
- raise ValueError
- except ValueError:
- optparser.error('--user-map should specify JSON file with format {"original_user": "sf_user", ...}')
- finally:
- f.close()
-
- import_options['user_map'] = user_map
-
- cli = AlluraImportApiClient(options.base_url, options.api_key, options.secret_key, options.verbose)
- doc_txt = open(doc_file_name).read()
-
- if options.wiki:
- import_wiki(cli, options.project, options.wiki, options, doc_txt)
-
-
-def import_wiki(cli, project, tool, options, doc_txt):
- url = '/rest/p/' + project + '/' + tool
- doc = json.loads(doc_txt)
- if 'wiki' in doc and 'default' in doc['wiki'] and 'artifacts' in doc['wiki']['default']:
- pages = doc['trackers']['default']['artifacts']
- else:
- pages = doc
- if options.verbose:
- print "Processing %d pages" % len(pages)
- for page in pages:
- title = page.pop('title').encode('utf-8')
- page['text'] = page['text'].encode('utf-8')
- page['labels'] = page['labels'].encode('utf-8')
- r = cli.call(url + '/' + title, **page)
- assert r == {}
- print 'Imported wiki page %s' % title
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5cd4382d/ForgeWiki/forgewiki/scripts/wiki_from_trac/wiki_from_trac.py
----------------------------------------------------------------------
diff --git a/ForgeWiki/forgewiki/scripts/wiki_from_trac/wiki_from_trac.py b/ForgeWiki/forgewiki/scripts/wiki_from_trac/wiki_from_trac.py
deleted file mode 100644
index 65630aa..0000000
--- a/ForgeWiki/forgewiki/scripts/wiki_from_trac/wiki_from_trac.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import argparse
-import logging
-from tempfile import NamedTemporaryFile
-from tg.decorators import cached_property
-
-from forgewiki.scripts.wiki_from_trac.extractors import WikiExporter
-from forgewiki.scripts.wiki_from_trac.loaders import load_data
-
-from allura.scripts import ScriptTask
-
-
-log = logging.getLogger(__name__)
-
-
-class WikiFromTrac(ScriptTask):
- """Import Trac Wiki to Allura Wiki"""
- @classmethod
- def parser(cls):
- parser = argparse.ArgumentParser(description='Import wiki from'
- 'Trac to allura wiki')
-
- parser.add_argument('trac_url', type=str, help='Trac URL')
- parser.add_argument('-a', '--api-ticket', dest='api_key', help='API ticket')
- parser.add_argument('-s', '--secret-key', dest='secret_key', help='Secret key')
- parser.add_argument('-p', '--project', dest='project', help='Project to import to')
- parser.add_argument('-t', '--tracker', dest='tracker', help='Tracker to import to')
- parser.add_argument('-f', '--forum', dest='forum', help='Forum tool to import to')
- parser.add_argument('-w', '--wiki', dest='wiki', help='Wiki tool to import to')
- parser.add_argument('-u', '--base-url', dest='base_url', default='https://sourceforge.net', help='Base Allura (%(default)s for default)')
- parser.add_argument('-o', dest='import_opts', default=[], action='append', help='Specify import option(s)', metavar='opt=val')
- parser.add_argument('--user-map', dest='user_map_file', help='Map original users to SF.net users', metavar='JSON_FILE')
- parser.add_argument('--validate', dest='validate', action='store_true', help='Validate import data')
- parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='Verbose operation')
- parser.add_argument('-c', '--continue', dest='cont', action='store_true', help='Continue import into existing tracker')
- parser.add_argument('-C', '--converter', dest='converter',
- default='html2text',
- help='Converter to use on wiki text. '
- 'Available options: '
- 'html2text (default) or regex')
-
- return parser
-
- @classmethod
- def execute(cls, options):
- with NamedTemporaryFile() as f:
- WikiExporter(options.trac_url, options).export(f)
- f.flush()
- load_data(f.name, cls.parser(), options)
-
-
-if __name__ == '__main__':
- WikiFromTrac.main()
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5cd4382d/scripts/allura_import.py
----------------------------------------------------------------------
diff --git a/scripts/allura_import.py b/scripts/allura_import.py
index b05d524..df92cd5 100644
--- a/scripts/allura_import.py
+++ b/scripts/allura_import.py
@@ -20,7 +20,7 @@ from optparse import OptionParser
from allura.lib.import_api import AlluraImportApiClient
from forgetracker.scripts.import_tracker import import_tracker
-from forgewiki.scripts.wiki_from_trac.loaders import import_wiki
+from tracwikiimporter.scripts.wiki_from_trac.loaders import import_wiki
def main():
http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5cd4382d/scripts/wiki-export.py
----------------------------------------------------------------------
diff --git a/scripts/wiki-export.py b/scripts/wiki-export.py
index 55baa04..3c51ed2 100755
--- a/scripts/wiki-export.py
+++ b/scripts/wiki-export.py
@@ -22,7 +22,7 @@ import json
import sys
from optparse import OptionParser
-from forgewiki.scripts.wiki_from_trac.extractors import WikiExporter
+from tracwikiimporter.scripts.wiki_from_trac.extractors import WikiExporter
def parse_options():
@@ -55,4 +55,4 @@ if __name__ == '__main__':
if options.out_filename:
out = open(options.out_filename, 'w')
- exporter.export(out)
\ No newline at end of file
+ exporter.export(out)