You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by st...@apache.org on 2016/04/29 20:38:56 UTC
svn commit: r1741682 [26/26] - in /subversion/branches/authzperf: ./ build/
build/ac-macros/ build/generator/ contrib/server-side/svncutter/ notes/
notes/api-errata/1.9/ notes/move-tracking/ subversion/
subversion/bindings/ctypes-python/ subversion/bin...
Modified: subversion/branches/authzperf/subversion/tests/svn_test_main.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/tests/svn_test_main.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/subversion/tests/svn_test_main.c (original)
+++ subversion/branches/authzperf/subversion/tests/svn_test_main.c Fri Apr 29 18:38:53 2016
@@ -407,26 +407,29 @@ do_test_num(const char *progname,
apr_pool_t *pool)
{
svn_boolean_t skip, xfail, wimp;
- svn_error_t *err = NULL;
+ svn_error_t *err;
const char *msg = NULL; /* the message this individual test prints out */
const struct svn_test_descriptor_t *desc;
const int array_size = get_array_size(test_funcs);
svn_boolean_t run_this_test; /* This test's mode matches DESC->MODE. */
enum svn_test_mode_t test_mode;
+ volatile int adjusted_num = test_num; /* volatile for setjmp */
+
+ /* This allows './some-test -- -1' to run the last test. */
+ if (adjusted_num < 0)
+ adjusted_num += array_size + 1;
/* Check our array bounds! */
- if (test_num < 0)
- test_num += array_size + 1;
- if ((test_num > array_size) || (test_num <= 0))
+ if ((adjusted_num > array_size) || (adjusted_num <= 0))
{
if (header_msg && *header_msg)
printf("%s", *header_msg);
- printf("FAIL: %s: THERE IS NO TEST NUMBER %2d\n", progname, test_num);
+ printf("FAIL: %s: THERE IS NO TEST NUMBER %2d\n", progname, adjusted_num);
skip_cleanup = TRUE;
return TRUE; /* BAIL, this test number doesn't exist. */
}
- desc = &test_funcs[test_num];
+ desc = &test_funcs[adjusted_num];
/* Check the test predicate. */
if (desc->predicate.func
&& desc->predicate.func(opts, desc->predicate.value, pool))
@@ -462,7 +465,7 @@ do_test_num(const char *progname,
{
/* Do test */
if (msg_only || skip || !run_this_test)
- ; /* pass */
+ err = NULL; /* pass */
else if (desc->func2)
err = (*desc->func2)(pool);
else
@@ -480,7 +483,7 @@ do_test_num(const char *progname,
}
/* Failure means unexpected results -- FAIL or XPASS. */
- skip_cleanup = log_results(progname, test_num, msg_only, run_this_test,
+ skip_cleanup = log_results(progname, adjusted_num, msg_only, run_this_test,
skip, xfail, wimp, err, msg, desc);
return skip_cleanup;
@@ -760,6 +763,23 @@ svn_test__init_auth_baton(svn_auth_baton
return SVN_NO_ERROR;
}
+
+svn_error_t *
+svn_test_make_sandbox_dir(const char **sb_dir_p,
+ const char *sb_name,
+ apr_pool_t *pool)
+{
+ const char *sb_dir;
+
+ sb_dir = svn_test_data_path(sb_name, pool);
+ SVN_ERR(svn_io_remove_dir2(sb_dir, TRUE, NULL, NULL, pool));
+ SVN_ERR(svn_io_make_dir_recursively(sb_dir, pool));
+ svn_test_add_dir_cleanup(sb_dir);
+
+ *sb_dir_p = sb_dir;
+
+ return SVN_NO_ERROR;
+}
/* Standard svn test program */
int
Modified: subversion/branches/authzperf/tools/backup/hot-backup.py.in
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/backup/hot-backup.py.in?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/backup/hot-backup.py.in (original)
+++ subversion/branches/authzperf/tools/backup/hot-backup.py.in Fri Apr 29 18:38:53 2016
@@ -70,7 +70,7 @@ def chmod_tree(path, mode, mask):
def safe_rmtree(dirname, retry=0):
"Remove the tree at DIRNAME, making it writable first"
def rmtree(dirname):
- chmod_tree(dirname, 0666, 0666)
+ chmod_tree(dirname, 0o666, 0o666)
shutil.rmtree(dirname)
if not os.path.exists(dirname):
@@ -117,7 +117,7 @@ try:
"num-backups=",
"verify",
"help"])
-except getopt.GetoptError, e:
+except getopt.GetoptError as e:
sys.stderr.write("ERROR: %s\n\n" % e)
sys.stderr.flush()
usage(sys.stderr)
@@ -137,6 +137,11 @@ for o, a in opts:
usage()
sys.exit()
+if archive_type not in (None, 'bz2', 'gz', 'zip', 'zip64'):
+ sys.stderr.write("ERROR: Bad --archive-type\n")
+ usage(sys.stderr)
+ sys.exit(2)
+
if len(args) != 2:
sys.stderr.write("ERROR: only two arguments allowed.\n\n")
sys.stderr.flush()
@@ -198,7 +203,7 @@ def get_youngest_revision():
"""Examine the repository REPO_DIR using the svnlook binary
specified by SVNLOOK, and return the youngest revision."""
- p = subprocess.Popen([svnlook, 'youngest', repo_dir],
+ p = subprocess.Popen([svnlook, 'youngest', '--', repo_dir],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
@@ -226,7 +231,7 @@ print("Beginning hot backup of '"+ repo_
try:
youngest = get_youngest_revision()
-except Exception, e:
+except Exception as e:
sys.stderr.write("%s\n" % e)
sys.stderr.flush()
sys.exit(1)
@@ -245,7 +250,7 @@ backup_subdir = os.path.join(backup_dir,
# rather than start from 1 and increment because the starting
# increments may have already been removed due to num_backups.
-regexp = re.compile("^" + repo + "-" + youngest +
+regexp = re.compile("^" + re.escape(repo) + "-" + re.escape(youngest) +
"(-(?P<increment>[0-9]+))?" + ext_re + "$")
directory_list = os.listdir(backup_dir)
young_list = [x for x in directory_list if regexp.search(x)]
@@ -262,8 +267,8 @@ if young_list:
### copied last.
print("Backing up repository to '" + backup_subdir + "'...")
-err_code = subprocess.call([svnadmin, "hotcopy", repo_dir,
- backup_subdir, "--clean-logs"])
+err_code = subprocess.call([svnadmin, "hotcopy", "--clean-logs",
+ '--', repo_dir, backup_subdir])
if err_code != 0:
sys.stderr.write("Unable to backup the repository.\n")
sys.stderr.flush()
@@ -274,7 +279,7 @@ else:
### Step 4: Verify the hotcopy
if verify_copy:
print("Verifying backup...")
- err_code = subprocess.call([svnadmin, "verify", "--quiet", backup_subdir])
+ err_code = subprocess.call([svnadmin, "verify", "--quiet", '--', backup_subdir])
if err_code != 0:
sys.stderr.write("Backup verification failed.\n")
sys.stderr.flush()
@@ -294,10 +299,10 @@ if archive_type:
tar = tarfile.open(archive_path, 'w:' + archive_type)
tar.add(backup_subdir, os.path.basename(backup_subdir))
tar.close()
- except ImportError, e:
+ except ImportError as e:
err_msg = "Import failed: " + str(e)
err_code = -2
- except tarfile.TarError, e:
+ except tarfile.TarError as e:
err_msg = "Tar failed: " + str(e)
err_code = -3
@@ -320,10 +325,10 @@ if archive_type:
for dirpath, dirs, files in os.walk(backup_subdir):
add_to_zip(zp, backup_dir, dirpath, dirs + files)
zp.close()
- except ImportError, e:
+ except ImportError as e:
err_msg = "Import failed: " + str(e)
err_code = -4
- except zipfile.error, e:
+ except zipfile.error as e:
err_msg = "Zip failed: " + str(e)
err_code = -5
@@ -340,7 +345,7 @@ if archive_type:
### NUM_BACKUPS.
if num_backups > 0:
- regexp = re.compile("^" + repo + "-[0-9]+(-[0-9]+)?" + ext_re + "$")
+ regexp = re.compile("^" + re.escape(repo) + "-[0-9]+(-[0-9]+)?" + ext_re + "$")
directory_list = os.listdir(backup_dir)
old_list = [x for x in directory_list if regexp.search(x)]
old_list.sort(comparator)
Modified: subversion/branches/authzperf/tools/buildbot/slaves/svn-sparc-solaris/svncheck.sh
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/buildbot/slaves/svn-sparc-solaris/svncheck.sh?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/buildbot/slaves/svn-sparc-solaris/svncheck.sh (original)
+++ subversion/branches/authzperf/tools/buildbot/slaves/svn-sparc-solaris/svncheck.sh Fri Apr 29 18:38:53 2016
@@ -28,7 +28,10 @@ cd ../obj
LD_PRELOAD_64=/export/home/wandisco/buildbot/install/lib/preloadable_libiconv.so
export LD_PRELOAD_64
-if [ $SVN_VER_MINOR -ge 9 ]; then
+if [ $SVN_VER_MINOR -ge 10 ]; then
+ echo "============ make svnserveautocheck"
+ make svnserveautocheck CLEANUP=1 PARALLEL=30 THREADED=1 GLOBAL_SCHEDULER=1 || exit $?
+elif [ $SVN_VER_MINOR -ge 9 ]; then
echo "============ make svnserveautocheck"
make svnserveautocheck CLEANUP=1 PARALLEL=30 THREADED=1 || exit $?
else
Modified: subversion/branches/authzperf/tools/buildbot/slaves/win32-SharpSvn/svntest-test.cmd
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/buildbot/slaves/win32-SharpSvn/svntest-test.cmd?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/buildbot/slaves/win32-SharpSvn/svntest-test.cmd (original)
+++ subversion/branches/authzperf/tools/buildbot/slaves/win32-SharpSvn/svntest-test.cmd Fri Apr 29 18:38:53 2016
@@ -30,6 +30,7 @@ SET ARGS=
SET FSFS=
SET LOCAL=
+SET RC=0
:next
IF "%1" == "-r" (
@@ -75,17 +76,23 @@ IF "%SVN_BRANCH%" LSS "1.9." (
IF "%LOCAL%+%FSFS%" == "1+1" (
echo win-tests.py -c %PARALLEL% %MODE% -f fsfs %ARGS% "%TESTDIR%\tests"
win-tests.py -c %PARALLEL% %MODE% -f fsfs %ARGS% "%TESTDIR%\tests"
- IF ERRORLEVEL 1 EXIT /B 1
+ IF ERRORLEVEL 1 SET RC=1
)
IF "%SVN%+%FSFS%" == "1+1" (
echo win-tests.py -c %PARALLEL% %MODE% -f fsfs -u svn://127.0.0.1 %ARGS% "%TESTDIR%\tests"
win-tests.py -c %PARALLEL% %MODE% -f fsfs -u svn://127.0.0.1 %ARGS% "%TESTDIR%\tests"
- IF ERRORLEVEL 1 EXIT /B 1
+ IF ERRORLEVEL 1 SET RC=1
)
IF "%DAV%+%FSFS%" == "1+1" (
- echo win-tests.py -c %PARALLEL% %MODE% -f fsfs --httpd-dir "%CD%\..\deps\release\httpd" --httpd-port %TESTPORT% -u http://127.0.0.1:%TESTPORT% %ARGS% "%TESTDIR%\tests"
- win-tests.py -c %PARALLEL% %MODE% -f fsfs --httpd-dir "%CD%\..\deps\release\httpd" --httpd-port %TESTPORT% -u http://127.0.0.1:%TESTPORT% %ARGS% "%TESTDIR%\tests"
- IF ERRORLEVEL 1 EXIT /B 1
+ echo win-tests.py -c %PARALLEL% %MODE% -f fsfs --httpd-no-log --httpd-dir "%CD%\..\deps\release\httpd" --httpd-port %TESTPORT% -u http://127.0.0.1:%TESTPORT% %ARGS% "%TESTDIR%\tests"
+ win-tests.py -c %PARALLEL% %MODE% -f fsfs --httpd-no-log --httpd-dir "%CD%\..\deps\release\httpd" --httpd-port %TESTPORT% -u http://127.0.0.1:%TESTPORT% %ARGS% "%TESTDIR%\tests"
+ IF ERRORLEVEL 1 SET RC=1
)
+
+IF EXIST "%TEMP%\svn-*" (
+ echo "SVN Files left:"
+ dir "%TEMP%"
+)
+EXIT /B %RC%
Modified: subversion/branches/authzperf/tools/client-side/bash_completion
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/client-side/bash_completion?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/client-side/bash_completion (original)
+++ subversion/branches/authzperf/tools/client-side/bash_completion Fri Apr 29 18:38:53 2016
@@ -1136,7 +1136,7 @@ _svnadmin ()
;;
dump)
cmdOpts="-r --revision --incremental -q --quiet --deltas \
- -M --memory-cache-size"
+ -M --memory-cache-size -F --file"
;;
freeze)
cmdOpts="-F --file"
@@ -1150,7 +1150,8 @@ _svnadmin ()
load)
cmdOpts="--ignore-uuid --force-uuid --parent-dir -q --quiet \
--use-pre-commit-hook --use-post-commit-hook \
- --bypass-prop-validation -M --memory-cache-size"
+ --bypass-prop-validation -M --memory-cache-size \
+ --no-flush-to-disk -F --file"
;;
lstxns)
cmdOpts="-r --revision"
Modified: subversion/branches/authzperf/tools/client-side/svn-graph.pl
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/client-side/svn-graph.pl?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/client-side/svn-graph.pl (original)
+++ subversion/branches/authzperf/tools/client-side/svn-graph.pl Fri Apr 29 18:38:53 2016
@@ -43,7 +43,6 @@ use Getopt::Std;
$|=1;
require SVN::Core;
-require SVN::Ra;
require SVN::Client;
# The URL of the Subversion repository we wish to graph
@@ -60,17 +59,6 @@ my $startpath;
# Set the variables declared above.
parse_commandline();
-# Point at the root of a repository so we get can look at
-# every revision.
-my $auth = (new SVN::Client())->auth;
-my $ra = SVN::Ra->new(url => $repos_url, auth => $auth);
-
-# Handle identifier for the aboslutely youngest revision.
-if ($youngest eq 'HEAD')
-{
- $youngest = $ra->get_latest_revnum();
-}
-
# The "interesting" nodes are potential sources for copies. This list
# grows as we move through time.
# The "tracking" nodes are the most recent revisions of paths we're
@@ -110,7 +98,7 @@ usage: svn-graph.pl [-r START_REV:END_RE
getopts('r:p:h', \%cmd_opts) or die $usage;
die $usage if scalar(@ARGV) < 1;
- $repos_url = $ARGV[0];
+ $repos_url = SVN::Core::uri_canonicalize($ARGV[0]);
$cmd_opts{'r'} =~ m/(\d+)(:(.+))?/;
if ($3)
@@ -207,6 +195,7 @@ sub process_revision
# Write a descriptor for the graph in GraphViz .dot format to stdout.
sub write_graph_descriptor
{
+ my $client = SVN::Client->new;
# Begin writing the graph descriptor.
print "digraph tree {\n";
print "\tgraph [bgcolor=white];\n";
@@ -215,7 +204,7 @@ sub write_graph_descriptor
print "\n";
# Retrieve the requested history.
- $ra->get_log(['/'], $startrev, $youngest, 0, 1, 0, \&process_revision);
+ $client->log($repos_url, $startrev, $youngest, 1, 0, \&process_revision);
# Now ensure that everything is linked.
foreach my $codeline_change (keys %codeline_changes_forward)
Modified: subversion/branches/authzperf/tools/client-side/svn-mergeinfo-normalizer/help-cmd.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/client-side/svn-mergeinfo-normalizer/help-cmd.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/client-side/svn-mergeinfo-normalizer/help-cmd.c (original)
+++ subversion/branches/authzperf/tools/client-side/svn-mergeinfo-normalizer/help-cmd.c Fri Apr 29 18:38:53 2016
@@ -50,8 +50,8 @@ svn_min__help(apr_getopt_t *os,
const char *config_path;
char help_header[] =
- N_("usage: svn <subcommand> [options] [args]\n"
- "Subversion svn:mergeinfo normalization and reduction tool.\n"
+ N_("usage: svn-mergeinfo-normalizer <subcommand> [options] [args]\n"
+ "Subversion mergeinfo normalization and reduction tool.\n"
"Type 'svn-mergeinfo-normalizer help <subcommand>' for help on a specific\n"
"subcommand. Type 'svn-mergeinfo-normalizer --version' to see the program\n"
"version and RA modules or 'svn-mergeinfo-normalizer --version --quiet'\n"
Modified: subversion/branches/authzperf/tools/client-side/svn-mergeinfo-normalizer/logic.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/client-side/svn-mergeinfo-normalizer/logic.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/client-side/svn-mergeinfo-normalizer/logic.c (original)
+++ subversion/branches/authzperf/tools/client-side/svn-mergeinfo-normalizer/logic.c Fri Apr 29 18:38:53 2016
@@ -305,7 +305,7 @@ show_removing_obsoletes(svn_min__opt_sta
&& progress->needs_header)
{
SVN_ERR(svn_cmdline_printf(scratch_pool,
- _("\n Removing obsolete entries ...\n")));
+ _("\n Trying to remove obsolete entries ...\n")));
progress->needs_header = FALSE;
}
@@ -498,9 +498,11 @@ find_surviving_copies(apr_array_header_t
* enabled in it.
*
* If LOCAL_ONLY is set, only remove branches that are known to have been
- * deleted as per LOOKUP - this is for quick checks. Track progress in
- * PROGRESS and update MERGEINFO is we can remove the info for branch PATH
- * from it.
+ * deleted (as per LOOKUP) with no surviving copies etc. This is for quick
+ * checks.
+ *
+ * Track progress in PROGRESS and update MERGEINFO if we can remove the
+ * info for branch PATH from it.
*
* Use SCRATCH_POOL for temporaries.
*/
@@ -1339,10 +1341,10 @@ show_elision_result(svn_mergeinfo_t pare
if (parent_mergeinfo)
SVN_ERR(svn_cmdline_printf(scratch_pool,
_("\n Sub-tree merge info cannot be elided due to "
- "the following branches:\n")));
+ "the following branch(es):\n")));
else
SVN_ERR(svn_cmdline_printf(scratch_pool,
- _("\n Merge info kept for the following branches:\n")));
+ _("\n Merge info kept for the following branch(es):\n")));
sorted_mi = svn_sort__hash(subtree_mergeinfo,
svn_sort_compare_items_lexically,
@@ -1606,7 +1608,7 @@ show_obsoletes_summary(svn_min__branch_l
iterpool = svn_pool_create(scratch_pool);
SVN_ERR(svn_cmdline_printf(iterpool,
- _("\nEncountered %d missing branches:\n"),
+ _("\nEncountered %d missing branch(es):\n"),
paths->nelts));
for (i = 0; i < paths->nelts; ++i)
{
Modified: subversion/branches/authzperf/tools/client-side/svn-vendor.py
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/client-side/svn-vendor.py?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/client-side/svn-vendor.py (original)
+++ subversion/branches/authzperf/tools/client-side/svn-vendor.py Fri Apr 29 18:38:53 2016
@@ -200,7 +200,7 @@ class FSO(object):
'''
def __init__(self):
self.wc_path = None
- self.state = [ "-", "-" ] # '-': absent, 'F': file, 'D': dir
+ self.state = [ "-", "-" ] # '-': absent, 'F': file, 'D': dir, 'L': symlink
def status(self):
return "[%s%s]" % (self.state[S_WC], self.state[S_IM])
@@ -216,8 +216,8 @@ class FSOCollection(dict):
Collection of FSOs
'''
def print(self):
- print(" / Status in working copy (-:absent, F:file, D:dir)")
- print(" |/ Status in imported sources (-:absent, F:file, D:dir)")
+ print(" / Status in working copy (-:absent, F:file, D:dir, L:link)")
+ print(" |/ Status in imported sources (-:absent, F:file, D:dir, L:link)")
for k in sorted(self.keys(), key=filename_sort_key):
e = self[k]
print("%s %s%s" % (e.status(), shlex.quote(k),
@@ -310,7 +310,8 @@ class Config(dict):
def print(self):
for k in sorted(self):
o = self[k]
- print("# %s" % o.helpmsg)
+ for s in o.helpmsg.split('\n'):
+ print("# %s" % s)
print("%-20s: %s" % (k, str(o)))
print()
@@ -330,6 +331,11 @@ class SvnVndImport(cmd.Cmd):
self.importdir = importdir
self.svninfo = svninfo
self.config = Config()
+ self.config.add_option('symlink-handling',
+ ConfigOpt("as-is", "How symbolic links are handled;\n" +
+ " 'dereference' treats as normal files/dirs (and " +
+ "ignores dangling links);\n" +
+ " 'as-is' imports as symlinks"))
self.config.add_option('save-diff-copied',
ConfigOpt(None, "Save 'svn diff' output on the " +
"moved/copied files and directories to this " +
@@ -359,12 +365,13 @@ class SvnVndImport(cmd.Cmd):
def scan(self):
self.items = FSOCollection()
self.info(1, "Scanning working copy directory...")
- self.get_lists(self.wcdir, S_WC)
+ self.get_lists(self.wcdir, S_WC, False)
self.info(1, "Scanning imported directory...")
- self.get_lists(self.importdir, S_IM)
+ self.get_lists(self.importdir, S_IM,
+ self.config.get('symlink-handling') == "dereference")
- def get_lists(self, top, where):
- for d, dn, fn in os.walk(top, followlinks=True):
+ def get_lists(self, top, where, deref):
+ for d, dn, fn in os.walk(top, followlinks=deref):
dr = os.path.relpath(d, top)
# If under .svn directory at the top (SVN 1.7+) or has .svn
# in the path (older SVN), ignore
@@ -374,9 +381,36 @@ class SvnVndImport(cmd.Cmd):
continue
if dr != '.':
self.items.add(dr, where, "D")
- for f in fn:
+ dnn = [] # List where we'll descend
+ for f in fn + dn:
fr = os.path.normpath(os.path.join(dr, f))
- self.items.add(fr, where, "F")
+ frp = os.path.join(d, f)
+ if os.path.islink(frp):
+ if deref:
+ # Dereferencing:
+ # - check for dangling/absolute/out-of-tree symlinks and abort
+ rl = os.readlink(frp)
+ if not os.path.exists(frp):
+ self.info(1, "WARN: Ignoring dangling symlink %s -> %s" % (fr, rl))
+ continue
+ if os.path.isabs(rl):
+ self.info(1, "WARN: Ignoring absolute symlink %s -> %s" % (fr, rl))
+ continue
+ tgt = os.path.normpath(os.path.join(dr, rl))
+ if tgt == ".." or tgt.startswith(".." + os.sep):
+ self.info(1, "WARN: Ignoring out-of-wc symlink %s -> %s" % (fr, rl))
+ continue
+ else:
+ # Importing symlinks as-is, no need to check.
+ self.items.add(fr, where, "L")
+ continue
+ # If we get here, treat symlinks to files as regular files, and add directories
+ # to the list of traversed subdirs
+ if os.path.isfile(frp):
+ self.items.add(fr, where, "F")
+ if os.path.isdir(frp):
+ dnn.append(f)
+ dn[:] = dnn
def onecmd(self, str):
'Override for checking number of arguments'
@@ -403,6 +437,10 @@ class SvnVndImport(cmd.Cmd):
pos = 0
atatime = 100
output = ""
+ # svn treats '@' specially (peg revision); if there's such character in a
+ # file name - append an empty peg revision
+ args_fixed = list(map(lambda x: x + "@" if x.find("@") != -1 else x, args_fixed))
+ args_split = list(map(lambda x: x + "@" if x.find("@") != -1 else x, args_split))
while pos < len(args_split) or (pos == 0 and len(args_split) == 0):
svnargs = ['svn'] + args_fixed + args_split[pos : pos + atatime]
pos += atatime
@@ -558,6 +596,10 @@ class SvnVndImport(cmd.Cmd):
if xsrc.state[S_WC] != xdst.state[S_IM]:
# Different kinds - definitely not the same object
return 0
+ if xsrc.state[S_WC] == "L" or xdst.state[S_IM] == "L":
+ # Symlinks are not considered the same object (same target in
+ # different dirs refers to different objects).
+ return 0
if xsrc.state[S_WC] == "D":
return self.similarity_dir(src, dst, threshold, lst_removal)
else:
@@ -734,8 +776,10 @@ class SvnVndImport(cmd.Cmd):
edit and load.
'''
self.parse_args(arg, 0, "detect")
+ # Configurable for file/dirs; symlinks are never similar.
self.detect({ "D": self.config.get('dir-similarity'),
- "F": self.config.get('file-similarity')})
+ "F": self.config.get('file-similarity'),
+ "L": 1001 })
def do_apply(self, arg):
'''
@@ -782,10 +826,44 @@ class SvnVndImport(cmd.Cmd):
os.path.join(self.wcdir, i))
files_added.append(i)
flg = "(added file)"
+ elif nk_im == "L":
+ tim = os.readlink(os.path.join(self.importdir, i))
+ os.symlink(tim, os.path.join(self.wcdir, i))
+ files_added.append(i)
+ flg = "(added symlink)"
else:
# Not in imported sources, not in WC (moved
# away/removed) - nothing to do
pass
+ elif nk_wc == "L":
+ # Symbolic link in a working copy
+ if nk_im == "L":
+ # Symbolic link in both. If the same target, do nothing. Otherwise,
+ # replace.
+ twc = os.readlink(os.path.join(self.wcdir, i))
+ tim = os.readlink(os.path.join(self.importdir, i))
+ if tim != twc:
+ self.run_svn(["rm", "--force", i])
+ os.symlink(tim, os.path.join(self.wcdir, i))
+ files_added.append(i)
+ flg = "(replaced symlink)"
+ elif nk_im == "D":
+ # Was a symlink, now a directory. Replace.
+ self.run_svn(["rm", "--force", i])
+ os.mkdir(os.path.join(self.wcdir, i))
+ dirs_added.append(i)
+ flg = "(replaced symlink with dir)"
+ elif nk_im == "F":
+ # Symlink replaced with file.
+ self.run_svn(["rm", "--force", i])
+ shutil.copyfile(os.path.join(self.importdir, i),
+ os.path.join(self.wcdir, i))
+ files_added.append(i)
+ flg = "(replaced symlink with file)"
+ else:
+ # Was a symlink, removed
+ files_removed.append(i)
+ flg = "(removed symlink)"
elif nk_wc == "F":
# File in a working copy
if nk_im == "D":
@@ -799,6 +877,13 @@ class SvnVndImport(cmd.Cmd):
shutil.copyfile(os.path.join(self.importdir, i),
os.path.join(self.wcdir, i))
flg = "(copied)"
+ elif nk_im == "L":
+ # Was a file, now a symlink. Replace.
+ self.run_svn(["rm", "--force", i])
+ tim = os.readlink(os.path.join(self.importdir, i))
+ os.symlink(tim, os.path.join(self.wcdir, i))
+ files_added.append(i)
+ flg = "(replaced file with symlink)"
else:
# Was a file, removed
files_removed.append(i)
@@ -817,6 +902,13 @@ class SvnVndImport(cmd.Cmd):
os.path.join(self.wcdir, i))
files_added.append(i)
flg = "(replaced dir with file)"
+ elif nk_im == "L":
+ # Was a directory, now a symlink. Replace.
+ self.run_svn(["rm", "--force", i])
+ tim = os.readlink(os.path.join(self.importdir, i))
+ os.symlink(tim, os.path.join(self.wcdir, i))
+ files_added.append(i)
+ flg = "(replaced dir with symlink)"
else:
# Directory removed
dirs_removed.append(i)
Propchange: subversion/branches/authzperf/tools/dev/svnmover/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Fri Apr 29 18:38:53 2016
@@ -0,0 +1 @@
+svnmover
Modified: subversion/branches/authzperf/tools/dev/which-error.py
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/dev/which-error.py?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/dev/which-error.py (original)
+++ subversion/branches/authzperf/tools/dev/which-error.py Fri Apr 29 18:38:53 2016
@@ -95,6 +95,8 @@ def print_error(code):
except KeyError:
if code == -41:
print("Sit by a lake.")
+ elif code >= 120100 and code < 121000:
+ print('%08d <error code from libserf; see serf.h>' % (code))
else:
print('%08d *** UNKNOWN ERROR CODE ***' % (code))
Modified: subversion/branches/authzperf/tools/dev/x509-parser.c
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/dev/x509-parser.c?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/dev/x509-parser.c (original)
+++ subversion/branches/authzperf/tools/dev/x509-parser.c Fri Apr 29 18:38:53 2016
@@ -94,7 +94,8 @@ get_der_cert_from_stream(const svn_strin
apr_pool_t *pool)
{
svn_string_t *raw;
- SVN_ERR(svn_string_from_stream(&raw, in, pool, pool));
+ SVN_ERR(svn_string_from_stream2(&raw, in, SVN__STREAM_CHUNK_SIZE,
+ pool));
*der_cert = NULL;
Propchange: subversion/branches/authzperf/tools/dist/
------------------------------------------------------------------------------
--- svn:ignore (original)
+++ svn:ignore Fri Apr 29 18:38:53 2016
@@ -1,3 +1,4 @@
+svn-test-work
prefix
tempdir
deploy
Modified: subversion/branches/authzperf/tools/dist/backport.pl
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/dist/backport.pl?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/dist/backport.pl (original)
+++ subversion/branches/authzperf/tools/dist/backport.pl Fri Apr 29 18:38:53 2016
@@ -9,11 +9,11 @@ use v5.10.0; # needed for $^V
# experimental and "subject to change" in v5.18 (see perl5180delta). Every
# use of it now triggers a warning.
#
-# As of Perl v5.20.1, the semantics of given/when provided by Perl are
+# As of Perl v5.22.1, the semantics of given/when provided by Perl are
# compatible with those expected by the script, so disable the warning for
# those Perls. But don't try to disable the the warning category on Perls
# that don't know that category, since that breaks compilation.
-no if (v5.17.0 le $^V and $^V le v5.20.1),
+no if (v5.17.0 le $^V and $^V le v5.22.1),
warnings => 'experimental::smartmatch';
# Licensed to the Apache Software Foundation (ASF) under one
@@ -41,7 +41,7 @@ use File::Copy qw/copy move/;
use File::Temp qw/tempfile/;
use IO::Select ();
use IPC::Open3 qw/open3/;
-use POSIX qw/ctermid strftime isprint isspace/;
+use POSIX qw/ctermid strftime/;
use Text::Wrap qw/wrap/;
use Tie::File ();
@@ -51,6 +51,7 @@ use Tie::File ();
#
# TODO: document which are interpreted by sh and which should point to binary.
my $SVN = $ENV{SVN} || 'svn'; # passed unquoted to sh
+$SVN .= " --config-option=config:miscellany:log-encoding=UTF-8";
my $SHELL = $ENV{SHELL} // '/bin/sh';
my $VIM = 'vim';
my $EDITOR = $ENV{SVN_EDITOR} // $ENV{VISUAL} // $ENV{EDITOR} // 'ed';
@@ -113,7 +114,7 @@ my $STATUS = './STATUS';
my $STATEFILE = './.backports1';
my $BRANCHES = '^/subversion/branches';
my $TRUNK = '^/subversion/trunk';
-$ENV{LC_ALL} = "C"; # since we parse 'svn info' output and use isprint()
+$ENV{LC_ALL} = "C"; # since we parse 'svn info' output
# Globals.
my %ERRORS = ();
@@ -311,11 +312,11 @@ sub prompt {
ReadMode 'normal';
die if $@ or not defined $answer;
# Swallow terminal escape codes (e.g., arrow keys).
- unless (isprint $answer or isspace $answer) {
+ unless ($answer =~ m/^(?:[[:print:]]+|\s+)$/) {
$answer = (ReadKey -1) while defined $answer;
# TODO: provide an indication that the keystroke was sensed and ignored.
}
- } until defined $answer and (isprint $answer or isspace $answer);
+ } until defined $answer and ($answer =~ m/^(?:[[:print:]]+|\s+)$/);
print $answer;
return $answer;
};
Modified: subversion/branches/authzperf/tools/dist/backport/merger.py
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/dist/backport/merger.py?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/dist/backport/merger.py (original)
+++ subversion/branches/authzperf/tools/dist/backport/merger.py Fri Apr 29 18:38:53 2016
@@ -23,12 +23,14 @@ backport.merger - library for running ST
import backport.status
+import contextlib
import functools
import logging
import os
import re
import subprocess
import sys
+import tempfile
import time
import unittest
@@ -43,25 +45,26 @@ class UnableToMergeException(Exception):
pass
-def invoke_svn(argv, extra_env={}):
+def invoke_svn(argv):
"Run svn with ARGV as argv[1:]. Return (exit_code, stdout, stderr)."
# TODO(interactive mode): disable --non-interactive
child_env = os.environ.copy()
- child_env.update(extra_env)
- child = subprocess.Popen([SVN, '--non-interactive'] + argv,
+ child_env.update({'LC_ALL': 'C'})
+ argv = [SVN, '--non-interactive', '--config-option=config:miscellany:log-encoding=UTF-8'] + argv
+ child = subprocess.Popen(argv,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=child_env)
stdout, stderr = child.communicate()
return child.returncode, stdout.decode('UTF-8'), stderr.decode('UTF-8')
-def run_svn(argv, expected_stderr=None, extra_env={'LC_ALL': 'C'}):
+def run_svn(argv, expected_stderr=None):
"""Run svn with ARGV as argv[1:]. If EXPECTED_STDERR is None, raise if the
exit code is non-zero or stderr is non-empty. Else, treat EXPECTED_STDERR as
a regexp, and ignore an errorful exit or stderr messages if the latter match
the regexp. Return exit_code, stdout, stderr."""
- exit_code, stdout, stderr = invoke_svn(argv, extra_env)
+ exit_code, stdout, stderr = invoke_svn(argv)
if exit_code == 0 and not stderr:
return exit_code, stdout, stderr
elif expected_stderr and re.compile(expected_stderr).search(stderr):
@@ -141,6 +144,14 @@ def _includes_only_svn_mergeinfo_changes
return False
+@contextlib.contextmanager
+def log_message_file(logmsg):
+ "Context manager that returns a file containing the text LOGMSG."
+ with tempfile.NamedTemporaryFile(mode='w+', encoding="UTF-8") as logmsg_file:
+ logmsg_file.write(logmsg)
+ logmsg_file.flush()
+ yield logmsg_file.name
+
def merge(entry, expected_stderr=None, *, commit=False):
"""Merges ENTRY into the working copy at cwd.
@@ -209,7 +220,9 @@ def merge(entry, expected_stderr=None, *
s = s[:-1]
open('./STATUS', 'w').write(s)
- run_svn_quiet(['commit', '-m', logmsg])
+ # Don't assume we can pass UTF-8 in argv.
+ with log_message_file(logmsg) as logmsg_filename:
+ run_svn_quiet(['commit', '-F', logmsg_filename])
# TODO(interactive mode): add the 'svn status' display
@@ -217,9 +230,9 @@ def merge(entry, expected_stderr=None, *
revnum = last_changed_revision('./STATUS')
if commit:
- # TODO: disable this for test runs
# Sleep to avoid out-of-order commit notifications
- time.sleep(15)
+ if not os.getenv("SVN_BACKPORT_DONT_SLEEP"): # enabled by the test suite
+ time.sleep(15)
second_logmsg = "Remove the {!r} branch, {} in r{}."\
.format(entry.branch, reintegrated_word, revnum)
run_svn(['rm', '-m', second_logmsg, '--', branch_url])
Modified: subversion/branches/authzperf/tools/dist/backport_tests.py
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/dist/backport_tests.py?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/dist/backport_tests.py (original)
+++ subversion/branches/authzperf/tools/dist/backport_tests.py Fri Apr 29 18:38:53 2016
@@ -646,6 +646,22 @@ def backport_STATUS_mods(sbox):
expected_stdout, expected_stderr)
#----------------------------------------------------------------------
+@BackportTest('76cee987-25c9-4d6c-ad40-000000000012')
+def backport_unicode_entry(sbox):
+ "an entry containing literal UTF-8"
+
+ # r6: nominate r4
+ approved_entries = [
+ make_entry([4], notes="Hello ðº"),
+ ]
+ sbox.simple_append(STATUS, serialize_STATUS(approved_entries))
+ sbox.simple_commit(message='Nominate r4')
+
+ # Run it.
+ run_backport(sbox)
+
+
+#----------------------------------------------------------------------
########################################################################
# Run the tests
@@ -663,11 +679,14 @@ test_list = [ None,
backport_branch_with_original_revision,
backport_otherproject_change,
backport_STATUS_mods,
+ backport_unicode_entry,
# When adding a new test, include the test number in the last
- # 6 bytes of the UUID.
+ # 6 bytes of the UUID, in decimal.
]
if __name__ == '__main__':
+ # Using putenv() here is fine because this file is never run as a module.
+ os.putenv('SVN_BACKPORT_DONT_SLEEP', '1')
svntest.main.run_tests(test_list)
# NOTREACHED
Modified: subversion/branches/authzperf/tools/dist/detect-conflicting-backports.py
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/dist/detect-conflicting-backports.py?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/dist/detect-conflicting-backports.py (original)
+++ subversion/branches/authzperf/tools/dist/detect-conflicting-backports.py Fri Apr 29 18:38:53 2016
@@ -69,7 +69,7 @@ if sys.argv[1:]:
sys.exit(0)
backport.merger.no_local_mods('./STATUS')
-sf = backport.status.StatusFile(open('./STATUS'))
+sf = backport.status.StatusFile(open('./STATUS', encoding="UTF-8"))
ERRORS = collections.defaultdict(list)
Modified: subversion/branches/authzperf/tools/dist/merge-approved-backports.py
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/dist/merge-approved-backports.py?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/dist/merge-approved-backports.py (original)
+++ subversion/branches/authzperf/tools/dist/merge-approved-backports.py Fri Apr 29 18:38:53 2016
@@ -40,7 +40,7 @@ if sys.argv[1:]:
sys.exit(0)
backport.merger.no_local_mods('./STATUS')
-sf = backport.status.StatusFile(open('./STATUS'))
+sf = backport.status.StatusFile(open('./STATUS', encoding="UTF-8"))
# Duplicate sf.paragraphs, since merge() will be removing elements from it.
entries_paras = list(sf.entries_paras())
Modified: subversion/branches/authzperf/tools/dist/security/_gnupg.py
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/tools/dist/security/_gnupg.py?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/tools/dist/security/_gnupg.py (original)
+++ subversion/branches/authzperf/tools/dist/security/_gnupg.py Fri Apr 29 18:38:53 2016
@@ -1,29 +1,3 @@
-# Copyright (c) 2008-2011 by Vinay Sajip.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# * Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-# * The name(s) of the copyright holder(s) may not be used to endorse or
-# promote products derived from this software without specific prior
-# written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) "AS IS" AND ANY EXPRESS OR
-# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
-# EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
-# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
-# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
-# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
""" A wrapper for the 'gpg' command::
Portions of this module are derived from A.M. Kuchling's well-designed
@@ -53,14 +27,14 @@ Vinay Sajip to make use of the subproces
and so does not work on Windows). Renamed to gnupg.py to avoid confusion with
the previous versions.
-Modifications Copyright (C) 2008-2011 Vinay Sajip. All rights reserved.
+Modifications Copyright (C) 2008-2014 Vinay Sajip. All rights reserved.
A unittest harness (test_gnupg.py) has also been added.
"""
-import locale
+__version__ = "0.3.8.dev0"
__author__ = "Vinay Sajip"
-__date__ = "$02-Sep-2011 13:18:12$"
+__date__ = "$07-Dec-2014 18:46:17$"
try:
from io import StringIO
@@ -71,12 +45,20 @@ import codecs
import locale
import logging
import os
+import re
import socket
from subprocess import Popen
from subprocess import PIPE
import sys
import threading
+STARTUPINFO = None
+if os.name == 'nt':
+ try:
+ from subprocess import STARTUPINFO, STARTF_USESHOWWINDOW, SW_HIDE
+ except ImportError:
+ STARTUPINFO = None
+
try:
import logging.NullHandler as NullHandler
except ImportError:
@@ -86,13 +68,61 @@ except ImportError:
try:
unicode
_py3k = False
+ string_types = basestring
+ text_type = unicode
except NameError:
_py3k = True
+ string_types = str
+ text_type = str
logger = logging.getLogger(__name__)
if not logger.handlers:
logger.addHandler(NullHandler())
+# We use the test below because it works for Jython as well as CPython
+if os.path.__name__ == 'ntpath':
+ # On Windows, we don't need shell quoting, other than worrying about
+ # paths with spaces in them.
+ def shell_quote(s):
+ return '"%s"' % s
+else:
+ # Section copied from sarge
+
+ # This regex determines which shell input needs quoting
+ # because it may be unsafe
+ UNSAFE = re.compile(r'[^\w%+,./:=@-]')
+
+ def shell_quote(s):
+ """
+ Quote text so that it is safe for Posix command shells.
+
+ For example, "*.py" would be converted to "'*.py'". If the text is
+ considered safe it is returned unquoted.
+
+ :param s: The value to quote
+ :type s: str (or unicode on 2.x)
+ :return: A safe version of the input, from the point of view of Posix
+ command shells
+ :rtype: The passed-in type
+ """
+ if not isinstance(s, string_types):
+ raise TypeError('Expected string type, got %s' % type(s))
+ if not s:
+ result = "''"
+ elif not UNSAFE.search(s):
+ result = s
+ else:
+ result = "'%s'" % s.replace("'", r"'\''")
+ return result
+
+ # end of sarge code
+
+# Now that we use shell=False, we shouldn't need to quote arguments.
+# Use no_quote instead of shell_quote to remind us of where quoting
+# was needed.
+def no_quote(s):
+ return s
+
def _copy_data(instream, outstream):
# Copy one stream to another
sent = 0
@@ -102,7 +132,7 @@ def _copy_data(instream, outstream):
enc = 'ascii'
while True:
data = instream.read(1024)
- if len(data) == 0:
+ if not data:
break
sent += len(data)
logger.debug("sending chunk (%d): %r", sent, data[:256])
@@ -132,34 +162,58 @@ def _write_passphrase(stream, passphrase
passphrase = '%s\n' % passphrase
passphrase = passphrase.encode(encoding)
stream.write(passphrase)
- logger.debug("Wrote passphrase: %r", passphrase)
+ logger.debug('Wrote passphrase')
def _is_sequence(instance):
- return isinstance(instance,list) or isinstance(instance,tuple)
+ return isinstance(instance, (list, tuple, set, frozenset))
-def _make_binary_stream(s, encoding):
+def _make_memory_stream(s):
try:
- if _py3k:
- if isinstance(s, str):
- s = s.encode(encoding)
- else:
- if type(s) is not str:
- s = s.encode(encoding)
from io import BytesIO
rv = BytesIO(s)
except ImportError:
rv = StringIO(s)
return rv
+def _make_binary_stream(s, encoding):
+ if _py3k:
+ if isinstance(s, str):
+ s = s.encode(encoding)
+ else:
+ if type(s) is not str:
+ s = s.encode(encoding)
+ return _make_memory_stream(s)
+
class Verify(object):
"Handle status messages for --verify"
+ TRUST_UNDEFINED = 0
+ TRUST_NEVER = 1
+ TRUST_MARGINAL = 2
+ TRUST_FULLY = 3
+ TRUST_ULTIMATE = 4
+
+ TRUST_LEVELS = {
+ "TRUST_UNDEFINED" : TRUST_UNDEFINED,
+ "TRUST_NEVER" : TRUST_NEVER,
+ "TRUST_MARGINAL" : TRUST_MARGINAL,
+ "TRUST_FULLY" : TRUST_FULLY,
+ "TRUST_ULTIMATE" : TRUST_ULTIMATE,
+ }
+
def __init__(self, gpg):
self.gpg = gpg
self.valid = False
self.fingerprint = self.creation_date = self.timestamp = None
self.signature_id = self.key_id = None
self.username = None
+ self.key_status = None
+ self.status = None
+ self.pubkey_fingerprint = None
+ self.expire_timestamp = None
+ self.sig_timestamp = None
+ self.trust_text = None
+ self.trust_level = None
def __nonzero__(self):
return self.valid
@@ -167,14 +221,31 @@ class Verify(object):
__bool__ = __nonzero__
def handle_status(self, key, value):
- if key in ("TRUST_UNDEFINED", "TRUST_NEVER", "TRUST_MARGINAL",
- "TRUST_FULLY", "TRUST_ULTIMATE", "RSA_OR_IDEA", "NODATA",
- "IMPORT_RES", "PLAINTEXT", "PLAINTEXT_LENGTH"):
+ if key in self.TRUST_LEVELS:
+ self.trust_text = key
+ self.trust_level = self.TRUST_LEVELS[key]
+ elif key in ("RSA_OR_IDEA", "NODATA", "IMPORT_RES", "PLAINTEXT",
+ "PLAINTEXT_LENGTH", "POLICY_URL", "DECRYPTION_INFO",
+ "DECRYPTION_OKAY", "INV_SGNR", "FILE_START", "FILE_ERROR",
+ "FILE_DONE", "PKA_TRUST_GOOD", "PKA_TRUST_BAD", "BADMDC",
+ "GOODMDC", "NO_SGNR", "NOTATION_NAME", "NOTATION_DATA",
+ "PROGRESS", "PINENTRY_LAUNCHED", "NEWSIG"):
pass
elif key == "BADSIG":
self.valid = False
self.status = 'signature bad'
self.key_id, self.username = value.split(None, 1)
+ elif key == "ERRSIG":
+ self.valid = False
+ (self.key_id,
+ algo, hash_algo,
+ cls,
+ self.timestamp) = value.split()[:5]
+ self.status = 'signature error'
+ elif key == "EXPSIG":
+ self.valid = False
+ self.status = 'signature expired'
+ self.key_id, self.username = value.split(None, 1)
elif key == "GOODSIG":
self.valid = True
self.status = 'signature good'
@@ -190,28 +261,33 @@ class Verify(object):
elif key == "SIG_ID":
(self.signature_id,
self.creation_date, self.timestamp) = value.split()
- elif key == "ERRSIG":
+ elif key == "DECRYPTION_FAILED":
self.valid = False
- (self.key_id,
- algo, hash_algo,
- cls,
- self.timestamp) = value.split()[:5]
- self.status = 'signature error'
+ self.key_id = value
+ self.status = 'decryption failed'
elif key == "NO_PUBKEY":
self.valid = False
self.key_id = value
self.status = 'no public key'
- elif key in ("KEYEXPIRED", "SIGEXPIRED"):
+ elif key in ("KEYEXPIRED", "SIGEXPIRED", "KEYREVOKED"):
# these are useless in verify, since they are spit out for any
# pub/subkeys on the key, not just the one doing the signing.
# if we want to check for signatures with expired key,
- # the relevant flag is EXPKEYSIG.
+ # the relevant flag is EXPKEYSIG or REVKEYSIG.
pass
elif key in ("EXPKEYSIG", "REVKEYSIG"):
# signed with expired or revoked key
self.valid = False
self.key_id = value.split()[0]
- self.status = (('%s %s') % (key[:3], key[3:])).lower()
+ if key == "EXPKEYSIG":
+ self.key_status = 'signing key has expired'
+ else:
+ self.key_status = 'signing key was revoked'
+ self.status = self.key_status
+ elif key == "UNEXPECTED":
+ self.valid = False
+ self.key_id = value
+ self.status = 'unexpected data'
else:
raise ValueError("Unknown status message: %r" % key)
@@ -280,8 +356,8 @@ class ImportResult(object):
'problem': reason, 'text': self.problem_reason[reason]})
elif key == "IMPORT_RES":
import_res = value.split()
- for i in range(len(self.counts)):
- setattr(self, self.counts[i], int(import_res[i]))
+ for i, count in enumerate(self.counts):
+ setattr(self, count, int(import_res[i]))
elif key == "KEYEXPIRED":
self.results.append({'fingerprint': None,
'problem': '0', 'text': 'Key expired'})
@@ -293,12 +369,68 @@ class ImportResult(object):
def summary(self):
l = []
- l.append('%d imported'%self.imported)
+ l.append('%d imported' % self.imported)
if self.not_imported:
- l.append('%d not imported'%self.not_imported)
+ l.append('%d not imported' % self.not_imported)
return ', '.join(l)
-class ListKeys(list):
+ESCAPE_PATTERN = re.compile(r'\\x([0-9a-f][0-9a-f])', re.I)
+BASIC_ESCAPES = {
+ r'\n': '\n',
+ r'\r': '\r',
+ r'\f': '\f',
+ r'\v': '\v',
+ r'\b': '\b',
+ r'\0': '\0',
+}
+
+class SendResult(object):
+ def __init__(self, gpg):
+ self.gpg = gpg
+
+ def handle_status(self, key, value):
+ logger.debug('SendResult: %s: %s', key, value)
+
+class SearchKeys(list):
+ ''' Handle status messages for --search-keys.
+
+ Handle pub and uid (relating the latter to the former).
+
+ Don't care about the rest
+ '''
+
+ UID_INDEX = 1
+ FIELDS = 'type keyid algo length date expires'.split()
+
+ def __init__(self, gpg):
+ self.gpg = gpg
+ self.curkey = None
+ self.fingerprints = []
+ self.uids = []
+
+ def get_fields(self, args):
+ result = {}
+ for i, var in enumerate(self.FIELDS):
+ result[var] = args[i]
+ result['uids'] = []
+ return result
+
+ def pub(self, args):
+ self.curkey = curkey = self.get_fields(args)
+ self.append(curkey)
+
+ def uid(self, args):
+ uid = args[self.UID_INDEX]
+ uid = ESCAPE_PATTERN.sub(lambda m: chr(int(m.group(1), 16)), uid)
+ for k, v in BASIC_ESCAPES.items():
+ uid = uid.replace(k, v)
+ self.curkey['uids'].append(uid)
+ self.uids.append(uid)
+
+ def handle_status(self, key, value):
+ pass
+
+class ListKeys(SearchKeys):
''' Handle status messages for --list-keys.
Handle pub and uid (relating the latter to the former).
@@ -307,7 +439,6 @@ class ListKeys(list):
crt = X.509 certificate
crs = X.509 certificate and private key available
- sub = subkey (secondary key)
ssb = secret subkey (secondary key)
uat = user attribute (same as user id except for field 10).
sig = signature
@@ -316,24 +447,17 @@ class ListKeys(list):
grp = reserved for gpgsm
rvk = revocation key
'''
- def __init__(self, gpg):
- self.gpg = gpg
- self.curkey = None
- self.fingerprints = []
- self.uids = []
+
+ UID_INDEX = 9
+ FIELDS = 'type trust length algo keyid date expires dummy ownertrust uid'.split()
def key(self, args):
- vars = ("""
- type trust length algo keyid date expires dummy ownertrust uid
- """).split()
- self.curkey = {}
- for i in range(len(vars)):
- self.curkey[vars[i]] = args[i]
- self.curkey['uids'] = []
- if self.curkey['uid']:
- self.curkey['uids'].append(self.curkey['uid'])
- del self.curkey['uid']
- self.append(self.curkey)
+ self.curkey = curkey = self.get_fields(args)
+ if curkey['uid']:
+ curkey['uids'].append(curkey['uid'])
+ del curkey['uid']
+ curkey['subkeys'] = []
+ self.append(curkey)
pub = sec = key
@@ -341,14 +465,34 @@ class ListKeys(list):
self.curkey['fingerprint'] = args[9]
self.fingerprints.append(args[9])
- def uid(self, args):
- self.curkey['uids'].append(args[9])
- self.uids.append(args[9])
+ def sub(self, args):
+ subkey = [args[4], args[11]]
+ self.curkey['subkeys'].append(subkey)
+
+
+class ScanKeys(ListKeys):
+ ''' Handle status messages for --with-fingerprint.'''
+
+ def sub(self, args):
+ # --with-fingerprint --with-colons somehow outputs fewer colons,
+ # use the last value args[-1] instead of args[11]
+ subkey = [args[4], args[-1]]
+ self.curkey['subkeys'].append(subkey)
- def handle_status(self, key, value):
- pass
+class TextHandler(object):
+ def _as_text(self):
+ return self.data.decode(self.gpg.encoding, self.gpg.decode_errors)
+
+ if _py3k:
+ __str__ = _as_text
+ else:
+ __unicode__ = _as_text
+
+ def __str__(self):
+ return self.data
-class Crypt(Verify):
+
+class Crypt(Verify, TextHandler):
"Handle status messages for --encrypt and --decrypt"
def __init__(self, gpg):
Verify.__init__(self, gpg)
@@ -362,18 +506,17 @@ class Crypt(Verify):
__bool__ = __nonzero__
- def __str__(self):
- return self.data.decode(self.gpg.encoding, self.gpg.decode_errors)
-
def handle_status(self, key, value):
if key in ("ENC_TO", "USERID_HINT", "GOODMDC", "END_DECRYPTION",
- "BEGIN_SIGNING", "NO_SECKEY", "ERROR", "NODATA"):
+ "BEGIN_SIGNING", "NO_SECKEY", "ERROR", "NODATA", "PROGRESS",
+ "CARDCTRL", "BADMDC", "SC_OP_FAILURE", "SC_OP_SUCCESS",
+ "PINENTRY_LAUNCHED"):
# in the case of ERROR, this is because a more specific error
# message will have come first
pass
elif key in ("NEED_PASSPHRASE", "BAD_PASSPHRASE", "GOOD_PASSPHRASE",
"MISSING_PASSPHRASE", "DECRYPTION_FAILED",
- "KEY_NOT_CREATED"):
+ "KEY_NOT_CREATED", "NEED_PASSPHRASE_PIN"):
self.status = key.replace("_", " ").lower()
elif key == "NEED_PASSPHRASE_SYM":
self.status = 'need symmetric passphrase'
@@ -415,7 +558,8 @@ class GenKey(object):
return self.fingerprint or ''
def handle_status(self, key, value):
- if key in ("PROGRESS", "GOOD_PASSPHRASE", "NODATA"):
+ if key in ("PROGRESS", "GOOD_PASSPHRASE", "NODATA", "KEY_NOT_CREATED",
+ "PINENTRY_LAUNCHED"):
pass
elif key == "KEY_CREATED":
(self.type,self.fingerprint) = value.split()
@@ -434,7 +578,7 @@ class DeleteResult(object):
problem_reason = {
'1': 'No such key',
'2': 'Must delete secret key first',
- '3': 'Ambigious specification',
+ '3': 'Ambiguous specification',
}
def handle_status(self, key, value):
@@ -444,11 +588,18 @@ class DeleteResult(object):
else:
raise ValueError("Unknown status message: %r" % key)
-class Sign(object):
+ def __nonzero__(self):
+ return self.status == 'ok'
+
+ __bool__ = __nonzero__
+
+
+class Sign(TextHandler):
"Handle status messages for --sign"
def __init__(self, gpg):
self.gpg = gpg
self.type = None
+ self.hash_algo = None
self.fingerprint = None
def __nonzero__(self):
@@ -456,21 +607,26 @@ class Sign(object):
__bool__ = __nonzero__
- def __str__(self):
- return self.data.decode(self.gpg.encoding, self.gpg.decode_errors)
-
def handle_status(self, key, value):
if key in ("USERID_HINT", "NEED_PASSPHRASE", "BAD_PASSPHRASE",
- "GOOD_PASSPHRASE", "BEGIN_SIGNING"):
+ "GOOD_PASSPHRASE", "BEGIN_SIGNING", "CARDCTRL", "INV_SGNR",
+ "NO_SGNR", "MISSING_PASSPHRASE", "NEED_PASSPHRASE_PIN",
+ "SC_OP_FAILURE", "SC_OP_SUCCESS", "PROGRESS"):
pass
+ elif key in ("KEYEXPIRED", "SIGEXPIRED"):
+ self.status = 'key expired'
+ elif key == "KEYREVOKED":
+ self.status = 'key revoked'
elif key == "SIG_CREATED":
(self.type,
- algo, hashalgo, cls,
+ algo, self.hash_algo, cls,
self.timestamp, self.fingerprint
) = value.split()
else:
raise ValueError("Unknown status message: %r" % key)
+VERSION_RE = re.compile(r'gpg \(GnuPG\) (\d+(\.\d+)*)'.encode('ascii'), re.I)
+HEX_DIGITS_RE = re.compile(r'[0-9a-f]+$', re.I)
class GPG(object):
@@ -481,31 +637,54 @@ class GPG(object):
'delete': DeleteResult,
'generate': GenKey,
'import': ImportResult,
+ 'send': SendResult,
'list': ListKeys,
+ 'scan': ScanKeys,
+ 'search': SearchKeys,
'sign': Sign,
'verify': Verify,
}
"Encapsulate access to the gpg executable"
def __init__(self, gpgbinary='gpg', gnupghome=None, verbose=False,
- use_agent=False, keyring=None):
+ use_agent=False, keyring=None, options=None,
+ secret_keyring=None):
"""Initialize a GPG process wrapper. Options are:
gpgbinary -- full pathname for GPG binary.
gnupghome -- full pathname to where we can find the public and
private keyrings. Default is whatever gpg defaults to.
- keyring -- name of alternative keyring file to use. If specified,
- the default keyring is not used.
+ keyring -- name of alternative keyring file to use, or list of such
+ keyrings. If specified, the default keyring is not used.
+ options =-- a list of additional options to pass to the GPG binary.
+ secret_keyring -- name of alternative secret keyring file to use, or
+ list of such keyrings.
"""
self.gpgbinary = gpgbinary
self.gnupghome = gnupghome
+ if keyring:
+ # Allow passing a string or another iterable. Make it uniformly
+ # a list of keyring filenames
+ if isinstance(keyring, string_types):
+ keyring = [keyring]
self.keyring = keyring
+ if secret_keyring:
+ # Allow passing a string or another iterable. Make it uniformly
+ # a list of keyring filenames
+ if isinstance(secret_keyring, string_types):
+ secret_keyring = [secret_keyring]
+ self.secret_keyring = secret_keyring
self.verbose = verbose
self.use_agent = use_agent
- self.encoding = locale.getpreferredencoding()
- if self.encoding is None: # This happens on Jython!
- self.encoding = sys.stdin.encoding
+ if isinstance(options, str):
+ options = [options]
+ self.options = options
+ # Changed in 0.3.7 to use Latin-1 encoding rather than
+ # locale.getpreferredencoding falling back to sys.stdin.encoding
+ # falling back to utf-8, because gpg itself uses latin-1 as the default
+ # encoding.
+ self.encoding = 'latin-1'
if gnupghome and not os.path.isdir(self.gnupghome):
os.makedirs(self.gnupghome,0x1C0)
p = self._open_subprocess(["--version"])
@@ -514,25 +693,54 @@ class GPG(object):
if p.returncode != 0:
raise ValueError("Error invoking gpg: %s: %s" % (p.returncode,
result.stderr))
+ m = VERSION_RE.match(result.data)
+ if not m:
+ self.version = None
+ else:
+ dot = '.'.encode('ascii')
+ self.version = tuple([int(s) for s in m.groups()[0].split(dot)])
- def _open_subprocess(self, args, passphrase=False):
- # Internal method: open a pipe to a GPG subprocess and return
- # the file objects for communicating with it.
- cmd = [self.gpgbinary, '--status-fd 2 --no-tty']
+ def make_args(self, args, passphrase):
+ """
+ Make a list of command line elements for GPG. The value of ``args``
+ will be appended. The ``passphrase`` argument needs to be True if
+ a passphrase will be sent to GPG, else False.
+ """
+ cmd = [self.gpgbinary, '--status-fd', '2', '--no-tty']
if self.gnupghome:
- cmd.append('--homedir "%s" ' % self.gnupghome)
+ cmd.extend(['--homedir', no_quote(self.gnupghome)])
if self.keyring:
- cmd.append('--no-default-keyring --keyring "%s" ' % self.keyring)
+ cmd.append('--no-default-keyring')
+ for fn in self.keyring:
+ cmd.extend(['--keyring', no_quote(fn)])
+ if self.secret_keyring:
+ for fn in self.secret_keyring:
+ cmd.extend(['--secret-keyring', no_quote(fn)])
if passphrase:
- cmd.append('--batch --passphrase-fd 0')
+ cmd.extend(['--batch', '--passphrase-fd', '0'])
if self.use_agent:
cmd.append('--use-agent')
+ if self.options:
+ cmd.extend(self.options)
cmd.extend(args)
- cmd = ' '.join(cmd)
+ return cmd
+
+ def _open_subprocess(self, args, passphrase=False):
+ # Internal method: open a pipe to a GPG subprocess and return
+ # the file objects for communicating with it.
+ cmd = self.make_args(args, passphrase)
if self.verbose:
- print(cmd)
+ pcmd = ' '.join(cmd)
+ print(pcmd)
logger.debug("%s", cmd)
- return Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ if not STARTUPINFO:
+ si = None
+ else:
+ si = STARTUPINFO()
+ si.dwFlags = STARTF_USESHOWWINDOW
+ si.wShowWindow = SW_HIDE
+ return Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE,
+ startupinfo=si)
def _read_response(self, stream, result):
# Internal method: reads all the stderr output from GPG, taking notice
@@ -609,7 +817,7 @@ class GPG(object):
stderr.close()
stdout.close()
- def _handle_io(self, args, file, result, passphrase=None, binary=False):
+ def _handle_io(self, args, fileobj, result, passphrase=None, binary=False):
"Handle a call to GPG - pass input data, collect output data"
# Handle a basic data call - pass data to GPG, handle the output
# including status information. Garbage In, Garbage Out :)
@@ -620,7 +828,7 @@ class GPG(object):
stdin = p.stdin
if passphrase:
_write_passphrase(stdin, passphrase, self.encoding)
- writer = _threaded_copy_data(file, stdin)
+ writer = _threaded_copy_data(fileobj, stdin)
self._collect_output(p, result, writer, stdin)
return result
@@ -634,8 +842,15 @@ class GPG(object):
f.close()
return result
+ def set_output_without_confirmation(self, args, output):
+ "If writing to a file which exists, avoid a confirmation message."
+ if os.path.exists(output):
+ # We need to avoid an overwrite confirmation message
+ args.extend(['--batch', '--yes'])
+ args.extend(['--output', output])
+
def sign_file(self, file, keyid=None, passphrase=None, clearsign=True,
- detach=False, binary=False):
+ detach=False, binary=False, output=None):
"""sign file"""
logger.debug("sign_file: %s", file)
if binary:
@@ -649,7 +864,10 @@ class GPG(object):
elif clearsign:
args.append("--clearsign")
if keyid:
- args.append('--default-key "%s"' % keyid)
+ args.extend(['--default-key', no_quote(keyid)])
+ if output: # write the output to a file with the specified name
+ self.set_output_without_confirmation(args, output)
+
result = self.result_map['sign'](self)
#We could use _handle_io here except for the fact that if the
#passphrase is bad, gpg bails and you can't write the message.
@@ -701,8 +919,8 @@ class GPG(object):
logger.debug('Wrote to temp file: %r', s)
os.write(fd, s)
os.close(fd)
- args.append(fn)
- args.append('"%s"' % data_filename)
+ args.append(no_quote(fn))
+ args.append(no_quote(data_filename))
try:
p = self._open_subprocess(args)
self._collect_output(p, result, stdin=p.stdin)
@@ -710,6 +928,15 @@ class GPG(object):
os.unlink(fn)
return result
+ def verify_data(self, sig_filename, data):
+ "Verify the signature in sig_filename against data in memory"
+ logger.debug('verify_data: %r, %r ...', sig_filename, data[:16])
+ result = self.result_map['verify'](self)
+ args = ['--verify', no_quote(sig_filename), '-']
+ stream = _make_memory_stream(data)
+ self._handle_io(args, stream, result, binary=True)
+ return result
+
#
# KEY MANAGEMENT
#
@@ -773,7 +1000,8 @@ class GPG(object):
>>> import shutil
>>> shutil.rmtree("keys")
>>> gpg = GPG(gnupghome="keys")
- >>> result = gpg.recv_keys('pgp.mit.edu', '3FF0DB166A7476EA')
+ >>> os.chmod('keys', 0x1C0)
+ >>> result = gpg.recv_keys('keyserver.ubuntu.com', '92905378')
>>> assert result
"""
@@ -781,33 +1009,60 @@ class GPG(object):
logger.debug('recv_keys: %r', keyids)
data = _make_binary_stream("", self.encoding)
#data = ""
- args = ['--keyserver', keyserver, '--recv-keys']
- args.extend(keyids)
+ args = ['--keyserver', no_quote(keyserver), '--recv-keys']
+ args.extend([no_quote(k) for k in keyids])
self._handle_io(args, data, result, binary=True)
logger.debug('recv_keys result: %r', result.__dict__)
data.close()
return result
+ def send_keys(self, keyserver, *keyids):
+ """Send a key to a keyserver.
+
+ Note: it's not practical to test this function without sending
+ arbitrary data to live keyservers.
+ """
+ result = self.result_map['send'](self)
+ logger.debug('send_keys: %r', keyids)
+ data = _make_binary_stream('', self.encoding)
+ #data = ""
+ args = ['--keyserver', no_quote(keyserver), '--send-keys']
+ args.extend([no_quote(k) for k in keyids])
+ self._handle_io(args, data, result, binary=True)
+ logger.debug('send_keys result: %r', result.__dict__)
+ data.close()
+ return result
+
def delete_keys(self, fingerprints, secret=False):
which='key'
if secret:
which='secret-key'
if _is_sequence(fingerprints):
- fingerprints = ' '.join(fingerprints)
- args = ['--batch --delete-%s "%s"' % (which, fingerprints)]
+ fingerprints = [no_quote(s) for s in fingerprints]
+ else:
+ fingerprints = [no_quote(fingerprints)]
+ args = ['--batch', '--delete-%s' % which]
+ args.extend(fingerprints)
result = self.result_map['delete'](self)
p = self._open_subprocess(args)
self._collect_output(p, result, stdin=p.stdin)
return result
- def export_keys(self, keyids, secret=False):
+ def export_keys(self, keyids, secret=False, armor=True, minimal=False):
"export the indicated keys. 'keyid' is anything gpg accepts"
which=''
if secret:
which='-secret-key'
if _is_sequence(keyids):
- keyids = ' '.join(['"%s"' % k for k in keyids])
- args = ["--armor --export%s %s" % (which, keyids)]
+ keyids = [no_quote(k) for k in keyids]
+ else:
+ keyids = [no_quote(keyids)]
+ args = ['--export%s' % which]
+ if armor:
+ args.append('--armor')
+ if minimal:
+ args.extend(['--export-options','export-minimal'])
+ args.extend(keyids)
p = self._open_subprocess(args)
# gpg --export produces no status-fd output; stdout will be
# empty in case of failure
@@ -817,6 +1072,27 @@ class GPG(object):
logger.debug('export_keys result: %r', result.data)
return result.data.decode(self.encoding, self.decode_errors)
+ def _get_list_output(self, p, kind):
+ # Get the response information
+ result = self.result_map[kind](self)
+ self._collect_output(p, result, stdin=p.stdin)
+ lines = result.data.decode(self.encoding,
+ self.decode_errors).splitlines()
+ valid_keywords = 'pub uid sec fpr sub'.split()
+ for line in lines:
+ if self.verbose:
+ print(line)
+ logger.debug("line: %r", line.rstrip())
+ if not line:
+ break
+ L = line.strip().split(':')
+ if not L:
+ continue
+ keyword = L[0]
+ if keyword in valid_keywords:
+ getattr(result, keyword)(L)
+ return result
+
def list_keys(self, secret=False):
""" list the keys currently in the keyring
@@ -837,25 +1113,58 @@ class GPG(object):
which='keys'
if secret:
which='secret-keys'
- args = "--list-%s --fixed-list-mode --fingerprint --with-colons" % (which,)
- args = [args]
+ args = ["--list-%s" % which, "--fixed-list-mode", "--fingerprint",
+ "--with-colons"]
+ p = self._open_subprocess(args)
+ return self._get_list_output(p, 'list')
+
+ def scan_keys(self, filename):
+ """
+ List details of an ascii armored or binary key file
+ without first importing it to the local keyring.
+
+ The function achieves this by running:
+ $ gpg --with-fingerprint --with-colons filename
+ """
+ args = ['--with-fingerprint', '--with-colons']
+ args.append(no_quote(filename))
p = self._open_subprocess(args)
+ return self._get_list_output(p, 'scan')
+
+ def search_keys(self, query, keyserver='pgp.mit.edu'):
+ """ search keyserver by query (using --search-keys option)
- # there might be some status thingumy here I should handle... (amk)
- # ...nope, unless you care about expired sigs or keys (stevegt)
+ >>> import shutil
+ >>> shutil.rmtree('keys')
+ >>> gpg = GPG(gnupghome='keys')
+ >>> os.chmod('keys', 0x1C0)
+ >>> result = gpg.search_keys('<vi...@hotmail.com>')
+ >>> assert result, 'Failed using default keyserver'
+ >>> keyserver = 'keyserver.ubuntu.com'
+ >>> result = gpg.search_keys('<vi...@hotmail.com>', keyserver)
+ >>> assert result, 'Failed using keyserver.ubuntu.com'
+
+ """
+ query = query.strip()
+ if HEX_DIGITS_RE.match(query):
+ query = '0x' + query
+ args = ['--fixed-list-mode', '--fingerprint', '--with-colons',
+ '--keyserver', no_quote(keyserver), '--search-keys',
+ no_quote(query)]
+ p = self._open_subprocess(args)
# Get the response information
- result = self.result_map['list'](self)
+ result = self.result_map['search'](self)
self._collect_output(p, result, stdin=p.stdin)
lines = result.data.decode(self.encoding,
self.decode_errors).splitlines()
- valid_keywords = 'pub uid sec fpr'.split()
+ valid_keywords = ['pub', 'uid']
for line in lines:
if self.verbose:
print(line)
- logger.debug("line: %r", line.rstrip())
- if not line:
- break
+ logger.debug('line: %r', line.rstrip())
+ if not line: # sometimes get blank lines on Windows
+ continue
L = line.strip().split(':')
if not L:
continue
@@ -876,7 +1185,7 @@ class GPG(object):
>>> assert not result
"""
- args = ["--gen-key --batch"]
+ args = ["--gen-key", "--batch"]
result = self.result_map['generate'](self)
f = _make_binary_stream(input, self.encoding)
self._handle_io(args, f, result, binary=True)
@@ -890,15 +1199,13 @@ class GPG(object):
parms = {}
for key, val in list(kwargs.items()):
key = key.replace('_','-').title()
- parms[key] = val
+ if str(val).strip(): # skip empty strings
+ parms[key] = val
parms.setdefault('Key-Type','RSA')
- parms.setdefault('Key-Length',1024)
+ parms.setdefault('Key-Length',2048)
parms.setdefault('Name-Real', "Autogenerated Key")
- parms.setdefault('Name-Comment', "Generated by gnupg.py")
- try:
- logname = os.environ['LOGNAME']
- except KeyError:
- logname = os.environ['USERNAME']
+ logname = (os.environ.get('LOGNAME') or os.environ.get('USERNAME') or
+ 'unspecified')
hostname = socket.gethostname()
parms.setdefault('Name-Email', "%s@%s" % (logname.replace(' ', '_'),
hostname))
@@ -939,23 +1246,30 @@ class GPG(object):
"Encrypt the message read from the file-like object 'file'"
args = ['--encrypt']
if symmetric:
+ # can't be False or None - could be True or a cipher algo value
+ # such as AES256
args = ['--symmetric']
+ if symmetric is not True:
+ args.extend(['--cipher-algo', no_quote(symmetric)])
+ # else use the default, currently CAST5
else:
- args = ['--encrypt']
+ if not recipients:
+ raise ValueError('No recipients specified with asymmetric '
+ 'encryption')
if not _is_sequence(recipients):
recipients = (recipients,)
for recipient in recipients:
- args.append('--recipient "%s"' % recipient)
- if armor: # create ascii-armored output - set to False for binary output
+ args.extend(['--recipient', no_quote(recipient)])
+ if armor: # create ascii-armored output - False for binary output
args.append('--armor')
if output: # write the output to a file with the specified name
- if os.path.exists(output):
- os.remove(output) # to avoid overwrite confirmation message
- args.append('--output "%s"' % output)
- if sign:
- args.append('--sign --default-key "%s"' % sign)
+ self.set_output_without_confirmation(args, output)
+ if sign is True:
+ args.append('--sign')
+ elif sign:
+ args.extend(['--sign', '--default-key', no_quote(sign)])
if always_trust:
- args.append("--always-trust")
+ args.append('--always-trust')
result = self.result_map['crypt'](self)
self._handle_io(args, file, result, passphrase=passphrase, binary=True)
logger.debug('encrypt result: %r', result.data)
@@ -983,9 +1297,6 @@ class GPG(object):
'hello'
>>> result = gpg.encrypt("hello again",print1)
>>> message = str(result)
- >>> result = gpg.decrypt(message)
- >>> result.status == 'need passphrase'
- True
>>> result = gpg.decrypt(message,passphrase='bar')
>>> result.status in ('decryption failed', 'bad passphrase')
True
@@ -995,9 +1306,6 @@ class GPG(object):
True
>>> str(result)
'hello again'
- >>> result = gpg.encrypt("signed hello",print2,sign=print1)
- >>> result.status == 'need passphrase'
- True
>>> result = gpg.encrypt("signed hello",print2,sign=print1,passphrase='foo')
>>> result.status == 'encryption ok'
True
@@ -1023,9 +1331,7 @@ class GPG(object):
output=None):
args = ["--decrypt"]
if output: # write the output to a file with the specified name
- if os.path.exists(output):
- os.remove(output) # to avoid overwrite confirmation message
- args.append('--output "%s"' % output)
+ self.set_output_without_confirmation(args, output)
if always_trust:
args.append("--always-trust")
result = self.result_map['crypt'](self)
Modified: subversion/branches/authzperf/win-tests.py
URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/win-tests.py?rev=1741682&r1=1741681&r2=1741682&view=diff
==============================================================================
--- subversion/branches/authzperf/win-tests.py (original)
+++ subversion/branches/authzperf/win-tests.py Fri Apr 29 18:38:53 2016
@@ -32,6 +32,7 @@ import filecmp
import shutil
import traceback
import logging
+import re
try:
# Python >=3.0
import configparser
@@ -83,6 +84,9 @@ def _usage_exit():
print(" --disable-http-v2 : Do not advertise support for HTTPv2 on server")
print(" --disable-bulk-updates : Disable bulk updates on HTTP server")
print(" --ssl-cert : Path to SSL server certificate to trust.")
+ print(" --https : Run Apache httpd with an https setup.")
+ print(" --http2 : Enable http2 in Apache Httpd (>= 2.4.17).")
+ print(" --global-scheduler : Enable global scheduler.")
print(" --exclusive-wc-locks : Enable exclusive working copy locks")
print(" --memcached-dir=DIR : Run memcached from dir")
print(" --memcached-server= : Enable usage of the specified memcached server")
@@ -129,11 +133,12 @@ gen_obj = gen_win_dependencies.GenDepend
opts, args = my_getopt(sys.argv[1:], 'hrdvqct:pu:f:',
['release', 'debug', 'verbose', 'quiet', 'cleanup',
'test=', 'url=', 'svnserve-args=', 'fs-type=', 'asp.net-hack',
- 'httpd-dir=', 'httpd-port=', 'httpd-daemon',
+ 'httpd-dir=', 'httpd-port=', 'httpd-daemon', 'https',
'httpd-server', 'http-short-circuit', 'httpd-no-log',
'disable-http-v2', 'disable-bulk-updates', 'help',
'fsfs-packing', 'fsfs-sharding=', 'javahl', 'swig=',
- 'list', 'enable-sasl', 'bin=', 'parallel',
+ 'list', 'enable-sasl', 'bin=', 'parallel', 'http2',
+ 'global-scheduler',
'config-file=', 'server-minor-version=', 'log-level=',
'log-to-stdout', 'mode-filter=', 'milestone-filter=',
'ssl-cert=', 'exclusive-wc-locks', 'memcached-server=',
@@ -144,6 +149,7 @@ if len(args) > 1:
# Interpret the options and set parameters
base_url, fs_type, verbose, cleanup = None, None, None, None
+global_scheduler = None
repo_loc = 'local repository.'
objdir = 'Debug'
log = 'tests.log'
@@ -154,6 +160,8 @@ run_httpd = None
httpd_port = None
httpd_service = None
httpd_no_log = None
+use_ssl = False
+use_http2 = False
http_short_circuit = False
advertise_httpv2 = True
http_bulk_updates = True
@@ -214,6 +222,10 @@ for opt, val in opts:
httpd_service = 1
elif opt == '--httpd-no-log':
httpd_no_log = 1
+ elif opt == '--https':
+ use_ssl = 1
+ elif opt == '--http2':
+ use_http2 = 1
elif opt == '--http-short-circuit':
http_short_circuit = True
elif opt == '--disable-http-v2':
@@ -226,6 +238,8 @@ for opt, val in opts:
fsfs_packing = 1
elif opt == '--javahl':
test_javahl = 1
+ elif opt == '--global-scheduler':
+ global_scheduler = 1
elif opt == '--swig':
if val not in ['perl', 'python', 'ruby']:
sys.stderr.write('Running \'%s\' swig tests not supported (yet).\n'
@@ -292,7 +306,12 @@ if run_httpd:
if not httpd_port:
httpd_port = random.randrange(1024, 30000)
if not base_url:
- base_url = 'http://localhost:' + str(httpd_port)
+ if use_ssl:
+ scheme = 'https'
+ else:
+ scheme = 'http'
+
+ base_url = '%s://localhost:%d' % (scheme, httpd_port)
if base_url:
repo_loc = 'remote repository ' + base_url + '.'
@@ -433,7 +452,9 @@ class Svnserve:
args = [self.name] + self.args
print('Starting %s %s' % (self.kind, self.name))
- self.proc = subprocess.Popen([self.path] + args[1:])
+ env = os.environ.copy()
+ env['SVN_DBG_STACKTRACES_TO_STDERR'] = 'y'
+ self.proc = subprocess.Popen([self.path] + args[1:], env=env)
def stop(self):
if self.proc is not None:
@@ -449,8 +470,9 @@ class Svnserve:
class Httpd:
"Run httpd for DAV tests"
- def __init__(self, abs_httpd_dir, abs_objdir, abs_builddir, httpd_port,
- service, no_log, httpv2, short_circuit, bulk_updates):
+ def __init__(self, abs_httpd_dir, abs_objdir, abs_builddir, abs_srcdir,
+ httpd_port, service, use_ssl, use_http2, no_log, httpv2,
+ short_circuit, bulk_updates):
self.name = 'apache.exe'
self.httpd_port = httpd_port
self.httpd_dir = abs_httpd_dir
@@ -488,12 +510,19 @@ class Httpd:
self.dontdothat_file = os.path.join(abs_builddir,
CMDLINE_TEST_SCRIPT_NATIVE_PATH,
'svn-test-work', 'dontdothat')
+ self.certfile = os.path.join(abs_builddir,
+ CMDLINE_TEST_SCRIPT_NATIVE_PATH,
+ 'svn-test-work', 'cert.pem')
+ self.certkeyfile = os.path.join(abs_builddir,
+ CMDLINE_TEST_SCRIPT_NATIVE_PATH,
+ 'svn-test-work', 'cert-key.pem')
self.httpd_config = os.path.join(self.root, 'httpd.conf')
self.httpd_users = os.path.join(self.root, 'users')
self.httpd_mime_types = os.path.join(self.root, 'mime.types')
self.httpd_groups = os.path.join(self.root, 'groups')
self.abs_builddir = abs_builddir
self.abs_objdir = abs_objdir
+ self.abs_srcdir = abs_srcdir
self.service_name = 'svn-test-httpd-' + str(httpd_port)
if self.service:
@@ -509,6 +538,9 @@ class Httpd:
self._create_mime_types_file()
self._create_dontdothat_file()
+ if use_ssl:
+ self._create_cert_files()
+
# Obtain version.
version_vals = gen_obj._libraries['httpd'].version.split('.')
self.httpd_ver = float('%s.%s' % (version_vals[0], version_vals[1]))
@@ -517,9 +549,10 @@ class Httpd:
fp = open(self.httpd_config, 'w')
# Limit the number of threads (default = 64)
- fp.write('<IfModule mpm_winnt.c>\n')
- fp.write('ThreadsPerChild 16\n')
- fp.write('</IfModule>\n')
+ if not use_http2:
+ fp.write('<IfModule mpm_winnt.c>\n')
+ fp.write('ThreadsPerChild 16\n')
+ fp.write('</IfModule>\n')
# Global Environment
fp.write('ServerRoot ' + self._quote(self.root) + '\n')
@@ -537,6 +570,10 @@ class Httpd:
fp.write('LogLevel Crit\n')
# Write LoadModule for minimal system module
+ if use_ssl:
+ fp.write(self._sys_module('ssl_module', 'mod_ssl.so'))
+ if use_http2:
+ fp.write(self._sys_module('http2_module', 'mod_http2.so'))
fp.write(self._sys_module('dav_module', 'mod_dav.so'))
if self.httpd_ver >= 2.3:
fp.write(self._sys_module('access_compat_module', 'mod_access_compat.so'))
@@ -561,6 +598,18 @@ class Httpd:
# And for mod_dontdothat
fp.write(self._svn_module('dontdothat_module', 'mod_dontdothat.so'))
+ if use_ssl:
+ fp.write('SSLEngine on\n')
+ fp.write('SSLProtocol All -SSLv2 -SSLv3\n')
+ fp.write('SSLCertificateFile %s\n' % self._quote(self.certfile))
+ fp.write('SSLCertificateKeyFile %s\n' % self._quote(self.certkeyfile))
+
+ if use_ssl and use_http2:
+ fp.write('Protocols h2 http/1.1\n')
+ elif use_http2:
+ fp.write('Protocols h2c http/1.1\n')
+ fp.write('H2Direct on\n')
+
# Don't handle .htaccess, symlinks, etc.
fp.write('<Directory />\n')
fp.write('AllowOverride None\n')
@@ -633,6 +682,34 @@ class Httpd:
fp.write('/ = deny\n')
fp.close()
+ def _create_cert_files(self):
+ "Create certificate files"
+ # The unix build uses certificates encoded in davautocheck.sh
+ # Let's just read them from there
+
+ sh_path = os.path.join(self.abs_srcdir, 'subversion', 'tests', 'cmdline',
+ 'davautocheck.sh')
+ sh = open(sh_path).readlines()
+
+ def cert_extract(lines, what):
+ r = []
+ pattern = r'cat\s*\>\s*' + re.escape(what) + r'\s*\<\<([A-Z_a-z0-9]+)'
+ exit_marker = None
+ for i in lines:
+ if exit_marker:
+ if i.startswith(exit_marker):
+ return r
+ r.append(i)
+ else:
+ m = re.match(pattern, i)
+ if m:
+ exit_marker = m.groups(1)
+
+ cert_file = cert_extract(sh, '"$SSL_CERTIFICATE_FILE"')
+ cert_key = cert_extract(sh, '"$SSL_CERTIFICATE_KEY_FILE"')
+ open(self.certfile, 'w').write(''.join(cert_file))
+ open(self.certkeyfile, 'w').write(''.join(cert_key))
+
def _sys_module(self, name, path):
full_path = os.path.join(self.httpd_dir, 'modules', path)
return 'LoadModule ' + name + " " + self._quote(full_path) + '\n'
@@ -940,11 +1017,14 @@ if not list_tests:
daemon = Svnserve(svnserve_args, objdir, abs_objdir, abs_builddir)
if run_httpd:
- daemon = Httpd(abs_httpd_dir, abs_objdir, abs_builddir, httpd_port,
- httpd_service, httpd_no_log,
- advertise_httpv2, http_short_circuit,
+ daemon = Httpd(abs_httpd_dir, abs_objdir, abs_builddir, abs_srcdir,
+ httpd_port, httpd_service, use_ssl, use_http2,
+ httpd_no_log, advertise_httpv2, http_short_circuit,
http_bulk_updates)
+ if use_ssl and not ssl_cert:
+ ssl_cert = daemon.certfile
+
# Start service daemon, if any
if daemon:
daemon.start()
@@ -998,6 +1078,7 @@ if not test_javahl and not test_swig:
opts, args = run_tests.create_parser().parse_args([])
opts.url = base_url
opts.fs_type = fs_type
+ opts.global_scheduler = global_scheduler
opts.http_library = 'serf'
opts.server_minor_version = server_minor_version
opts.cleanup = cleanup