You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2020/04/13 20:34:07 UTC
[allura] 15/22: [#8356] shlex.split helper for py2/3 compatibility
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8356
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 106ccb4560da6993ae4fc6f274dd007c22469c76
Author: Dave Brondsema <da...@brondsema.net>
AuthorDate: Fri Apr 10 11:36:12 2020 -0400
[#8356] shlex.split helper for py2/3 compatibility
---
Allura/allura/command/base.py | 3 +--
Allura/allura/lib/helpers.py | 11 ++++++-----
Allura/allura/lib/macro.py | 4 +---
Allura/allura/lib/solr.py | 12 ++----------
Allura/allura/scripts/delete_projects.py | 3 ++-
Allura/allura/scripts/scripttask.py | 11 ++---------
Allura/allura/tests/scripts/test_delete_projects.py | 7 ++++---
Allura/allura/tests/test_helpers.py | 2 ++
Allura/allura/tests/test_scripttask.py | 3 +--
9 files changed, 21 insertions(+), 35 deletions(-)
diff --git a/Allura/allura/command/base.py b/Allura/allura/command/base.py
index ac4c798..8f23468 100644
--- a/Allura/allura/command/base.py
+++ b/Allura/allura/command/base.py
@@ -19,7 +19,6 @@ from __future__ import unicode_literals
from __future__ import absolute_import
import os
import logging
-import shlex
import tg
from paste.script import command
@@ -44,7 +43,7 @@ def run_command(command, args):
mod = __import__(mod, fromlist=[str(cls)])
command = getattr(mod, cls)
command = command(command.__name__)
- arg_list = shlex.split(args or '')
+ arg_list = h.shlex_split(args or '')
try:
command.parser.parse_args(arg_list)
except SystemExit:
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index 5309fcb..65c9a89 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -940,13 +940,14 @@ def ming_config_from_ini(ini_path):
yield
+def shlex_split(string):
+ # py2/3 compatibility
+ return [six.ensure_text(s) for s in shlex.split(six.ensure_str(string))]
+
+
def split_select_field_options(field_options):
try:
- # shlex have problems with parsing unicode,
- # it's better to pass properly encoded byte-string
- field_options = shlex.split(field_options.encode('utf-8'))
- # convert splitted string back to unicode
- field_options = list(map(really_unicode, field_options))
+ field_options = shlex_split(field_options)
except ValueError:
field_options = field_options.split()
# After regular split field_options might contain a " characters,
diff --git a/Allura/allura/lib/macro.py b/Allura/allura/lib/macro.py
index e348bad..147b2bd 100644
--- a/Allura/allura/lib/macro.py
+++ b/Allura/allura/lib/macro.py
@@ -19,7 +19,6 @@ from __future__ import unicode_literals
from __future__ import absolute_import
import cgi
import random
-import shlex
import logging
import traceback
import six.moves.urllib.request, six.moves.urllib.error, six.moves.urllib.parse
@@ -65,8 +64,7 @@ class parse(object):
if s.startswith('quote '):
return '[[' + s[len('quote '):] + ']]'
try:
- parts = [six.text_type(x, 'utf-8')
- for x in shlex.split(s.encode('utf-8'))]
+ parts = h.shlex_split(s)
if not parts:
return '[[' + s + ']]'
macro = self._lookup_macro(parts[0])
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index 18c6bd3..d97141e 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -17,7 +17,6 @@
from __future__ import unicode_literals
from __future__ import absolute_import
-import shlex
import logging
from tg import config
@@ -25,6 +24,7 @@ from paste.deploy.converters import asbool
import pysolr
import six
+from allura.lib.helpers import shlex_split
log = logging.getLogger(__name__)
@@ -155,15 +155,7 @@ class MockSOLR(object):
q = '' # shlex will hang on None
# Parse query
preds = []
- if six.PY2:
- # shlex can't handle unicode in py2
- q_parts = [
- _.decode('latin-1')
- for _ in
- shlex.split(q.encode('latin-1'))
- ]
- else:
- q_parts = shlex.split(q)
+ q_parts = shlex_split(q)
if fq:
q_parts += fq
for part in q_parts:
diff --git a/Allura/allura/scripts/delete_projects.py b/Allura/allura/scripts/delete_projects.py
index 4c8e985..4fbf898 100644
--- a/Allura/allura/scripts/delete_projects.py
+++ b/Allura/allura/scripts/delete_projects.py
@@ -20,6 +20,7 @@ from __future__ import absolute_import
import argparse
import logging
+import six
from webob import Request
from ming.odm.odmsession import session
@@ -73,7 +74,7 @@ class DeleteProjects(ScriptTask):
parser = argparse.ArgumentParser(description='Completely delete projects')
parser.add_argument('projects', metavar='nbhd/project', type=str, nargs='+',
help='List of projects to delete in a form nbhd_prefix/shortname')
- parser.add_argument('-r', '--reason', type=lambda s: s.decode('utf8'),
+ parser.add_argument('-r', '--reason', type=lambda s: six.ensure_text(s),
help='Reason why these projects are being deleted')
parser.add_argument('--disable-users', action='store_true', default=False,
help='Disable all users belonging to groups Admin and Developer in these projects.')
diff --git a/Allura/allura/scripts/scripttask.py b/Allura/allura/scripts/scripttask.py
index 1281877..08fa9c8 100644
--- a/Allura/allura/scripts/scripttask.py
+++ b/Allura/allura/scripts/scripttask.py
@@ -48,11 +48,9 @@ from __future__ import unicode_literals
from __future__ import absolute_import
import argparse
import logging
-import shlex
-import sys
from allura.lib.decorators import task
-import six
+from allura.lib.helpers import shlex_split
log = logging.getLogger(__name__)
@@ -77,12 +75,7 @@ class ScriptTask(object):
@classmethod
def _execute_task(cls, arg_string):
try:
- if isinstance(arg_string, six.text_type):
- # shlex doesn't support unicode fully, so encode it http://stackoverflow.com/a/14219159/79697
- # decode has to happen in the arg parser (e.g. see delete_projects.py)
- arg_string = arg_string.encode('utf8')
- args_split = shlex.split(arg_string or '')
- options = cls.parser().parse_args(args_split)
+ options = cls.parser().parse_args(shlex_split(arg_string or ''))
except SystemExit:
raise Exception("Error parsing args: '%s'" % arg_string)
return cls.execute(options)
diff --git a/Allura/allura/tests/scripts/test_delete_projects.py b/Allura/allura/tests/scripts/test_delete_projects.py
index c7bf4d3..b18cdfa 100644
--- a/Allura/allura/tests/scripts/test_delete_projects.py
+++ b/Allura/allura/tests/scripts/test_delete_projects.py
@@ -1,3 +1,4 @@
+# coding=utf-8
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -123,12 +124,12 @@ class TestDeleteProjects(TestController):
p = M.Project.query.get(shortname=self.p_shortname)
pid = p._id
assert p is not None, 'Can not find project to delete'
- self.run_script(['-r', 'The Reason', 'p/{}'.format(p.shortname)])
+ self.run_script(['-r', 'The Reason¢¢', 'p/{}'.format(p.shortname)])
session(p).expunge(p)
p = M.Project.query.get(shortname=p.shortname)
assert p is None, 'Project is not deleted'
- log.info.assert_called_once_with('Purging %s Reason: %s', '/p/test-delete/', 'The Reason')
- post_event.assert_called_once_with('project_deleted', project_id=pid, reason='The Reason')
+ log.info.assert_called_once_with('Purging %s Reason: %s', '/p/test-delete/', 'The Reason¢¢')
+ post_event.assert_called_once_with('project_deleted', project_id=pid, reason='The Reason¢¢')
def _disable_users(self, disable):
dev = M.User.by_username('test-user')
diff --git a/Allura/allura/tests/test_helpers.py b/Allura/allura/tests/test_helpers.py
index 9ccdec9..60642cf 100644
--- a/Allura/allura/tests/test_helpers.py
+++ b/Allura/allura/tests/test_helpers.py
@@ -339,6 +339,8 @@ def test_split_select_field_options():
['test message', 'test2'])
assert_equals(h.split_select_field_options('"test message test2'),
['test', 'message', 'test2'])
+ assert_equals(h.split_select_field_options('abc ƒå∂ ººº'),
+ ['abc', 'ƒå∂', 'ººº'])
def test_notifications_disabled():
diff --git a/Allura/allura/tests/test_scripttask.py b/Allura/allura/tests/test_scripttask.py
index 1daa844..67702f9 100644
--- a/Allura/allura/tests/test_scripttask.py
+++ b/Allura/allura/tests/test_scripttask.py
@@ -34,8 +34,7 @@ class TestScriptTask(unittest.TestCase):
return cls._parser
self.cls = TestScriptTask
- @mock.patch('allura.scripts.scripttask.sys')
- def test_arg_parsing(self, sys):
+ def test_arg_parsing(self):
"Make sure string of args gets correctly tokenized."
parser = self.cls.parser()
self.cls._execute_task('--dir "My Dir"')