You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by as...@apache.org on 2012/11/24 21:29:48 UTC
svn commit: r1413258 [33/33] - in /subversion/branches/compressed-pristines:
./ build/ build/ac-macros/ build/generator/ build/generator/templates/
contrib/client-side/emacs/ contrib/server-side/fsfsfixer/ notes/
notes/directory-index/ subversion/ subv...
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/example.conf
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/example.conf?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/example.conf (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/example.conf Sat Nov 24 20:29:11 2012
@@ -3,6 +3,7 @@
[DEFAULT]
svnbin: /usr/local/bin/svn
streams: http://svn.example.org:2069/commits/xml
+hook: /usr/bin/true
## The values below are used by ConfigParser's interpolation syntax.
## See http://docs.python.org/library/configparser
Added: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/irkerbridge.py
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/irkerbridge.py?rev=1413258&view=auto
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/irkerbridge.py (added)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/irkerbridge.py Sat Nov 24 20:29:11 2012
@@ -0,0 +1,298 @@
+#!/usr/bin/env python
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# IrkerBridge - Bridge an SvnPubSub stream to Irker.
+
+# Example:
+# irkerbridge.py --daemon --pidfile pid --logfile log config
+#
+# For detailed option help use:
+# irkerbridge.py --help
+
+# It expects a config file that has the following parameters:
+# streams=url
+# Space separated list of URLs to streams.
+# This option should only be in the DEFAULT section, is ignored in
+# all other sections.
+# NOTE: At current svnpubsub.client only accepts hostname and port
+# combos so the path is ignored and /commits/xml is used.
+# irker=hostname:port
+# The hostname/port combination of the irker daemon. If port is
+# omitted it defaults to 6659. Irker is connected to over UDP.
+# match=What to use to decide if the commit should be sent to irker.
+# It consists of the repository UUID followed by a slash and a glob pattern.
+# The UUID may be replaced by a * to match all UUIDs. The glob pattern will
+# be matched against all of the dirs_changed. Both the UUID and the glob
+# pattern must match to send the message to irker.
+# to=url
+# Space separated list of URLs (any URL that Irker will accept) to
+# send the resulting message to. At current Irker only supports IRC.
+# template=string
+# A string to use to format the output. The string is a Python
+# string Template. The following variables are available:
+# $author, $rev, $date, $uuid, $log, $log, $log_firstline,
+# $log_firstparagraph, $dirs_changed, $dirs_count, $dirs_count_s,
+# $subdirs_count, $subdirs_count_s, $dirs_root
+# Most of them should be self explanatory. $dirs_count is the number of
+# entries in $dirs_changed, $dirs_count_s is a friendly string version,
+# $dirs_root is the common root of all the $dirs_changed, $subdirs_count
+# is the number of subdirs under the $dirs_root that changed,
+# $subdirs_root_s is a friendly string version. $log_firstparagraph cuts
+# the log message at the first blank line and replaces newlines with spaces.
+#
+# Within the config file you have sections. Any configuration option
+# missing from a given section is found in the [DEFAULT] section.
+#
+# Section names are arbitrary names that mean nothing to the bridge. Each
+# section other than the [DEFAULT] section consists of a configuration that
+# may match and send a message to irker to deliver. All matching sections
+# will generate a message.
+#
+# Interpolation of values within the config file is allowed by including
+# %(name)s within a value. For example I can reference the UUID of a repo
+# repeatedly by doing:
+# [DEFAULT]
+# ASF_REPO=13f79535-47bb-0310-9956-ffa450edef68
+#
+# [#commits]
+# match=%(ASF_REPO)s/
+#
+# You can HUP the process to reload the config file without restarting the
+# process. However, you cannot change the streams it is listening to without
+# restarting the process.
+#
+# TODO: Logging in a better way.
+
+# Messages longer than this will be truncated and ... added to the end such
+# that the resulting message is no longer than this:
+MAX_PRIVMSG = 400
+
+import os
+import sys
+import posixpath
+import socket
+import json
+import urlparse
+import optparse
+import ConfigParser
+import traceback
+import signal
+import re
+import fnmatch
+from string import Template
+
+# Packages that come with svnpubsub
+import svnpubsub.client
+import daemonize
+
+class Daemon(daemonize.Daemon):
+ def __init__(self, logfile, pidfile, bdec):
+ daemonize.Daemon.__init__(self, logfile, pidfile)
+
+ self.bdec = bdec
+
+ def setup(self):
+ # There is no setup which the parent needs to wait for.
+ pass
+
+ def run(self):
+ print 'irkerbridge started, pid=%d' % (os.getpid())
+
+ mc = svnpubsub.client.MultiClient(self.bdec.hostports,
+ self.bdec.commit,
+ self.bdec.event)
+ mc.run_forever()
+
+
+class BigDoEverythingClass(object):
+ def __init__(self, config, options):
+ self.config = config
+ self.options = options
+ self.hostports = []
+ for url in config.get_value('streams').split():
+ parsed = urlparse.urlparse(url.strip())
+ self.hostports.append((parsed.hostname, parsed.port or 80))
+
+ def locate_matching_configs(self, rev):
+ result = [ ]
+ for section in self.config.sections():
+ match = self.config.get(section, "match").split('/', 1)
+ if len(match) < 2:
+ # No slash so assume all paths
+ match.append('*')
+ match_uuid, match_path = match
+ if rev.uuid == match_uuid or match_uuid == "*":
+ for path in rev.dirs_changed:
+ if fnmatch.fnmatch(path, match_path):
+ result.append(section)
+ break
+ return result
+
+ def fill_in_extra_args(self, rev):
+ # Add entries to the rev object that are useful for
+ # formatting.
+ rev.log_firstline = rev.log.split("\n",1)[0]
+ rev.log_firstparagraph = re.split("\r?\n\r?\n",rev.log,1)[0]
+ rev.log_firstparagraph = re.sub("\r?\n"," ",rev.log_firstparagraph)
+ if rev.dirs_changed:
+ rev.dirs_root = posixpath.commonprefix(rev.dirs_changed)
+ rev.dirs_count = len(rev.dirs_changed)
+ if rev.dirs_count > 1:
+ rev.dirs_count_s = " (%d dirs)" %(rev.dirs_count)
+ else:
+ rev.dirs_count_s = ""
+
+ rev.subdirs_count = rev.dirs_count
+ if rev.dirs_root in rev.dirs_changed:
+ rev.subdirs_count -= 1
+ if rev.subdirs_count > 1:
+ rev.subdirs_count_s = " + %d subdirs" % (rev.subdirs_count)
+ else:
+ rev.subdirs_count_s = ""
+
+ def _send(self, irker, msg):
+ sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+ irker_list = irker.split(':')
+ if len(irker_list) < 2:
+ irker_list.append(6659)
+ json_msg = json.dumps(msg)
+ sock.sendto(json_msg, (irker_list[0],int(irker_list[1])))
+ if self.options.verbose:
+ print "SENT: %s to %s" % (json_msg, irker)
+
+ def join_all(self):
+ # Like self.commit(), but ignores self.config.get(section, "template").
+ for section in self.config.sections():
+ irker = self.config.get(section, "irker")
+ to_list = self.config.get(section, "to").split()
+ if not irker or not to_list:
+ continue
+ for to in to_list:
+ msg = {'to': to, 'privmsg': ''}
+ self._send(irker, msg)
+
+ def commit(self, host, port, rev):
+ if self.options.verbose:
+ print "RECV: from %s:%s" % (host, port)
+ print json.dumps(vars(rev), indent=2)
+
+ try:
+ config_sections = self.locate_matching_configs(rev)
+ if len(config_sections) > 0:
+ self.fill_in_extra_args(rev)
+ for section in config_sections:
+ irker = self.config.get(section, "irker")
+ to_list = self.config.get(section, "to").split()
+ template = self.config.get(section, "template")
+ if not irker or not to_list or not template:
+ continue
+ privmsg = Template(template).safe_substitute(vars(rev))
+ if len(privmsg) > MAX_PRIVMSG:
+ privmsg = privmsg[:MAX_PRIVMSG-3] + '...'
+ for to in to_list:
+ msg = {'to': to, 'privmsg': privmsg}
+ self._send(irker, msg)
+
+ except:
+ print "Unexpected error:"
+ traceback.print_exc()
+ sys.stdout.flush()
+ raise
+
+ def event(self, host, port, event_name):
+ if self.options.verbose or event_name != "ping":
+ print 'EVENT: %s from %s:%s' % (event_name, host, port)
+ sys.stdout.flush()
+
+
+
+class ReloadableConfig(ConfigParser.SafeConfigParser):
+ def __init__(self, fname):
+ ConfigParser.SafeConfigParser.__init__(self)
+
+ self.fname = fname
+ self.read(fname)
+
+ signal.signal(signal.SIGHUP, self.hangup)
+
+ def hangup(self, signalnum, frame):
+ self.reload()
+
+ def reload(self):
+ print "RELOAD: config file: %s" % self.fname
+ sys.stdout.flush()
+
+ # Delete everything. Just re-reading would overlay, and would not
+ # remove sections/options. Note that [DEFAULT] will not be removed.
+ for section in self.sections():
+ self.remove_section(section)
+
+ # Get rid of [DEFAULT]
+ self.remove_section(ConfigParser.DEFAULTSECT)
+
+ # Now re-read the configuration file.
+ self.read(self.fname)
+
+ def get_value(self, which):
+ return self.get(ConfigParser.DEFAULTSECT, which)
+
+
+def main(args):
+ parser = optparse.OptionParser(
+ description='An SvnPubSub client that bridges the data to irker.',
+ usage='Usage: %prog [options] CONFIG_FILE',
+ )
+ parser.add_option('--logfile',
+ help='filename for logging')
+ parser.add_option('--verbose', action='store_true',
+ help="enable verbose logging")
+ parser.add_option('--pidfile',
+ help="the process' PID will be written to this file")
+ parser.add_option('--daemon', action='store_true',
+ help='run as a background daemon')
+
+ options, extra = parser.parse_args(args)
+
+ if len(extra) != 1:
+ parser.error('CONFIG_FILE is requried')
+ config_file = os.path.abspath(extra[0])
+
+ if options.daemon:
+ if options.logfile:
+ logfile = os.path.abspath(options.logfile)
+ else:
+ parser.error('LOGFILE is required when running as a daemon')
+
+ if options.pidfile:
+ pidfile = os.path.abspath(options.pidfile)
+ else:
+ parser.error('PIDFILE is required when running as a daemon')
+
+
+ config = ReloadableConfig(config_file)
+ bdec = BigDoEverythingClass(config, options)
+
+ d = Daemon(logfile, pidfile, bdec)
+ if options.daemon:
+ d.daemonize_exit()
+ else:
+ d.foreground()
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub Sat Nov 24 20:29:11 2012
@@ -1,35 +1 @@
-#!/bin/sh
-#
-# PROVIDE: svnpubsub
-# REQUIRE: DAEMON
-# KEYWORD: shutdown
-
-. /etc/rc.subr
-
-name="svnpubsub"
-rcvar=`set_rcvar`
-
-load_rc_config $name
-
-#
-# DO NOT CHANGE THESE DEFAULT VALUES HERE
-# SET THEM IN THE /etc/rc.conf FILE
-#
-svnpubsub_enable=${svnpubsub_enable-"NO"}
-svnpubsub_user=${svnpubsub_user-"svn"}
-svnpubsub_group=${svnpubsub_group-"svn"}
-svnpubsub_reactor=${svnpubsub_reactor-"poll"}
-svnpubsub_pidfile=${svnpubsub_pidfile-"/var/run/svnpubsub/svnpubsub.pid"}
-pidfile="${svnpubsub_pidfile}"
-
-export PYTHON_EGG_CACHE="/home/svn/.python-eggs"
-
-command="/usr/local/bin/twistd"
-command_args="-y /usr/local/svnpubsub/svnpubsub.tac \
- --logfile=/var/log/vc/svnpubsub.log \
- --pidfile=${pidfile} \
- --uid=${svnpubsub_user} --gid=${svnpubsub_user} \
- -r${svnpubsub_reactor}"
-
-
-run_rc_command "$1"
+link svnpubsub.freebsd
\ No newline at end of file
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.debian
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.debian?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.debian (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.debian Sat Nov 24 20:29:11 2012
@@ -19,7 +19,7 @@ svnpubsub_pidfile=${svnpubsub_pidfile-"/
pidfile="${svnpubsub_pidfile}"
TWSITD_CMD="/usr/bin/twistd -y /opt/svnpubsub/svnpubsub.tac \
- --logfile=/var/bwlog/svnpubsub/svnpubsub.log \
+ --logfile=/var/log/svnpubsub/svnpubsub.log \
--pidfile=${pidfile} \
--uid=${svnpubsub_user} --gid=${svnpubsub_user} \
-r${svnpubsub_reactor}"
Added: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd?rev=1413258&view=auto
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd (added)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnpubsub.freebsd Sat Nov 24 20:29:11 2012
@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+# PROVIDE: svnpubsub
+# REQUIRE: DAEMON
+# KEYWORD: shutdown
+
+. /etc/rc.subr
+
+name="svnpubsub"
+rcvar=`set_rcvar`
+
+load_rc_config $name
+
+#
+# DO NOT CHANGE THESE DEFAULT VALUES HERE
+# SET THEM IN THE /etc/rc.conf FILE
+#
+svnpubsub_enable=${svnpubsub_enable-"NO"}
+svnpubsub_user=${svnpubsub_user-"svn"}
+svnpubsub_group=${svnpubsub_group-"svn"}
+svnpubsub_reactor=${svnpubsub_reactor-"poll"}
+svnpubsub_pidfile=${svnpubsub_pidfile-"/var/run/svnpubsub/svnpubsub.pid"}
+pidfile="${svnpubsub_pidfile}"
+
+export PYTHON_EGG_CACHE="/home/svn/.python-eggs"
+
+command="/usr/local/bin/twistd"
+command_args="-y /usr/local/svnpubsub/svnpubsub.tac \
+ --logfile=/var/log/vc/svnpubsub.log \
+ --pidfile=${pidfile} \
+ --uid=${svnpubsub_user} --gid=${svnpubsub_user} \
+ -r${svnpubsub_reactor}"
+
+
+run_rc_command "$1"
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub Sat Nov 24 20:29:11 2012
@@ -1,38 +1 @@
-#!/bin/sh
-#
-# PROVIDE: svnwcsub
-# REQUIRE: DAEMON
-# KEYWORD: shutdown
-
-. /etc/rc.subr
-
-name="svnwcsub"
-rcvar=`set_rcvar`
-
-load_rc_config $name
-
-#
-# DO NOT CHANGE THESE DEFAULT VALUES HERE
-# SET THEM IN THE /etc/rc.conf FILE
-#
-svnwcsub_enable=${svnwcsub_enable-"NO"}
-svnwcsub_user=${svnwcsub_user-"svnwc"}
-svnwcsub_group=${svnwcsub_group-"svnwc"}
-svnwcsub_pidfile=${svnwcsub_pidfile-"/var/run/svnwcsub/svnwcsub.pub"}
-svnwcsub_env="PYTHON_EGG_CACHE"
-svnwcsub_cmd_int=${svnwcsub_cmd_int-"python"}
-svnwcsub_config=${svnwcsub_config-"/etc/svnwcsub.conf"}
-svnwcsub_logfile=${svnwcsub_logfile-"/var/log/svnwcsub/svnwcsub.log"}
-pidfile="${svnwcsub_pidfile}"
-
-export PYTHON_EGG_CACHE="/var/run/svnwcsub"
-
-command="/usr/local/svnpubsub/svnwcsub.py"
-command_interpreter="/usr/local/bin/${svnwcsub_cmd_int}"
-command_args="--daemon \
- --logfile=${svnwcsub_logfile} \
- --pidfile=${pidfile} \
- --uid=${svnwcsub_user} --gid=${svnwcsub_group} \
- --umask=002 ${svnwcsub_config}"
-
-run_rc_command "$1"
+link svnwcsub.freebsd
\ No newline at end of file
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.debian
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.debian?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.debian (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.debian Sat Nov 24 20:29:11 2012
@@ -16,7 +16,7 @@ svnwcsub_user=${svnwcsub_user-"svnwc"}
svnwcsub_group=${svnwcsub_group-"svnwc"}
svnwcsub_pidfile=${svnwcsub_pidfile-"/var/run/svnwcsub.pid"}
svnwcsub_config=${svnwcsub_config-"/etc/svnwcsub.conf"}
-svnwcsub_logfile=${svnwcsub_logfile-"/var/bwlog/svnwcsub/svnwcsub.log"}
+svnwcsub_logfile=${svnwcsub_logfile-"/var/log/svnwcsub/svnwcsub.log"}
pidfile="${svnwcsub_pidfile}"
SVNWCSUB_CMD="/opt/svnpubsub/svnwcsub.py \
@@ -24,6 +24,7 @@ SVNWCSUB_CMD="/opt/svnpubsub/svnwcsub.py
--logfile=${svnwcsub_logfile} \
--pidfile=${pidfile} \
--uid=${svnwcsub_user} --gid=${svnwcsub_group} \
+ --umask=002 \
${svnwcsub_config} "
RETVAL=0
Added: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.freebsd
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.freebsd?rev=1413258&view=auto
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.freebsd (added)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.freebsd Sat Nov 24 20:29:11 2012
@@ -0,0 +1,39 @@
+#!/bin/sh
+#
+# PROVIDE: svnwcsub
+# REQUIRE: DAEMON
+# KEYWORD: shutdown
+
+. /etc/rc.subr
+
+name="svnwcsub"
+rcvar=`set_rcvar`
+
+load_rc_config $name
+
+#
+# DO NOT CHANGE THESE DEFAULT VALUES HERE
+# SET THEM IN THE /etc/rc.conf FILE
+#
+svnwcsub_enable=${svnwcsub_enable-"NO"}
+svnwcsub_user=${svnwcsub_user-"svnwc"}
+svnwcsub_group=${svnwcsub_group-"svnwc"}
+svnwcsub_pidfile=${svnwcsub_pidfile-"/var/run/svnwcsub/svnwcsub.pub"}
+svnwcsub_env="PYTHON_EGG_CACHE"
+svnwcsub_cmd_int=${svnwcsub_cmd_int-"python"}
+svnwcsub_config=${svnwcsub_config-"/etc/svnwcsub.conf"}
+svnwcsub_logfile=${svnwcsub_logfile-"/var/log/svnwcsub/svnwcsub.log"}
+pidfile="${svnwcsub_pidfile}"
+
+export PYTHON_EGG_CACHE="/var/run/svnwcsub"
+
+command="/usr/local/svnpubsub/svnwcsub.py"
+command_interpreter="/usr/local/bin/${svnwcsub_cmd_int}"
+command_args="--daemon \
+ --logfile=${svnwcsub_logfile} \
+ --pidfile=${pidfile} \
+ --uid=${svnwcsub_user} --gid=${svnwcsub_group} \
+ --umask=002 \
+ ${svnwcsub_config}"
+
+run_rc_command "$1"
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.solaris
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.solaris?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.solaris (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/rc.d/svnwcsub.solaris Sat Nov 24 20:29:11 2012
@@ -14,8 +14,8 @@ SVNWCSUB_CMD="/usr/local/svnpubsub/svnwc
--daemon \
--logfile=${svnwcsub_logfile} \
--pidfile=${pidfile} \
- --umask=002 \
--uid=${svnwcsub_user} --gid=${svnwcsub_group} \
+ --umask=002 \
${svnwcsub_config}"
RETVAL=0
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/client.py
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/client.py?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/client.py (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/client.py Sat Nov 24 20:29:11 2012
@@ -137,13 +137,13 @@ class XMLStreamHandler(xml.sax.handler.C
elif self.chars and self.rev:
value = self.chars.strip()
if name == 'path':
- self.rev.dirs_changed.append(value)
+ self.rev.dirs_changed.append(value.decode('unicode_escape'))
elif name == 'author':
- self.rev.author = value
+ self.rev.author = value.decode('unicode_escape')
elif name == 'date':
- self.rev.date = value
+ self.rev.date = value.decode('unicode_escape')
elif name == 'log':
- self.rev.log = value
+ self.rev.log = value.decode('unicode_escape')
# Toss out any accumulated characters for this element.
self.chars = ''
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/server.py
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/server.py?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/server.py (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnpubsub/server.py Sat Nov 24 20:29:11 2012
@@ -73,12 +73,15 @@ import time
class Revision:
def __init__(self, r):
+ # Don't escape the values; json handles binary values fine.
+ # ET will happily emit literal control characters (eg, NUL),
+ # thus creating invalid XML, so the XML code paths do escaping.
self.rev = r.get('revision')
self.repos = r.get('repos')
- self.dirs_changed = [x.encode('unicode_escape') for x in r.get('dirs_changed')]
- self.author = r.get('author').encode('unicode_escape')
- self.log = r.get('log').encode('unicode_escape')
- self.date = r.get('date').encode('unicode_escape')
+ self.dirs_changed = [x for x in r.get('dirs_changed')]
+ self.author = r.get('author')
+ self.log = r.get('log')
+ self.date = r.get('date')
def render_commit(self, format):
if format == "json":
@@ -90,13 +93,13 @@ class Revision:
'date': self.date}}) +","
elif format == "xml":
c = ET.Element('commit', {'repository': self.repos, 'revision': "%d" % (self.rev)})
- ET.SubElement(c, 'author').text = self.author
- ET.SubElement(c, 'date').text = self.date
- ET.SubElement(c, 'log').text = self.log
+ ET.SubElement(c, 'author').text = self.author.encode('unicode_escape')
+ ET.SubElement(c, 'date').text = self.date.encode('unicode_escape')
+ ET.SubElement(c, 'log').text = self.log.encode('unicode_escape')
d = ET.SubElement(c, 'dirs_changed')
for p in self.dirs_changed:
x = ET.SubElement(d, 'path')
- x.text = p
+ x.text = p.encode('unicode_escape')
str = ET.tostring(c, 'UTF-8') + "\n"
return str[39:]
else:
@@ -112,7 +115,7 @@ class Revision:
d = ET.SubElement(c, 'dirs_changed')
for p in self.dirs_changed:
x = ET.SubElement(d, 'path')
- x.text = p
+ x.text = p.encode('unicode_escape')
str = ET.tostring(c, 'UTF-8') + "\n"
return str[39:]
else:
Modified: subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnwcsub.py
URL: http://svn.apache.org/viewvc/subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnwcsub.py?rev=1413258&r1=1413257&r2=1413258&view=diff
==============================================================================
--- subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnwcsub.py (original)
+++ subversion/branches/compressed-pristines/tools/server-side/svnpubsub/svnwcsub.py Sat Nov 24 20:29:11 2012
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# encoding: UTF-8
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -29,6 +30,7 @@
# See svnwcsub.conf for more information on its contents.
#
+import errno
import subprocess
import threading
import sys
@@ -71,6 +73,22 @@ def svn_info(svnbin, env, path):
info[line[:idx]] = line[idx+1:].strip()
return info
+try:
+ import glob
+ glob.iglob
+ def is_emptydir(path):
+ # ### If the directory contains only dotfile children, this will readdir()
+ # ### the entire directory. But os.readdir() is not exposed to us...
+ for x in glob.iglob('%s/*' % path):
+ return False
+ for x in glob.iglob('%s/.*' % path):
+ return False
+ return True
+except (ImportError, AttributeError):
+ # Python â¤2.4
+ def is_emptydir(path):
+ # This will read the entire directory list to memory.
+ return not os.listdir(path)
class WorkingCopy(object):
def __init__(self, bdec, path, url):
@@ -106,7 +124,7 @@ class WorkingCopy(object):
def _get_match(self, svnbin, env):
### quick little hack to auto-checkout missing working copies
- if not os.path.isdir(self.path):
+ if not os.path.isdir(self.path) or is_emptydir(self.path):
logging.info("autopopulate %s from %s" % (self.path, self.url))
subprocess.check_call([svnbin, 'co', '-q',
'--non-interactive',
@@ -131,7 +149,8 @@ class BigDoEverythingClasss(object):
self.svnbin = config.get_value('svnbin')
self.env = config.get_env()
self.tracking = config.get_track()
- self.worker = BackgroundWorker(self.svnbin, self.env)
+ self.hook = config.get_value('hook')
+ self.worker = BackgroundWorker(self.svnbin, self.env, self.hook)
self.watch = [ ]
self.hostports = [ ]
@@ -150,7 +169,7 @@ class BigDoEverythingClasss(object):
# Add it to our watchers, and trigger an svn update.
logging.info("Watching WC at %s <-> %s" % (wc.path, wc.url))
self.watch.append(wc)
- self.worker.add_work(OP_UPDATE, wc)
+ self.worker.add_work(OP_BOOT, wc)
def _normalize_path(self, path):
if path[0] != '/':
@@ -182,11 +201,12 @@ class BigDoEverythingClasss(object):
# Start logging warnings if the work backlog reaches this many items
BACKLOG_TOO_HIGH = 20
+OP_BOOT = 'boot'
OP_UPDATE = 'update'
OP_CLEANUP = 'cleanup'
class BackgroundWorker(threading.Thread):
- def __init__(self, svnbin, env):
+ def __init__(self, svnbin, env, hook):
threading.Thread.__init__(self)
# The main thread/process should not wait for this thread to exit.
@@ -195,20 +215,28 @@ class BackgroundWorker(threading.Thread)
self.svnbin = svnbin
self.env = env
+ self.hook = hook
self.q = Queue.Queue()
self.has_started = False
def run(self):
while True:
- if self.q.qsize() > BACKLOG_TOO_HIGH:
- logging.warn('worker backlog is at %d', self.q.qsize())
-
# This will block until something arrives
operation, wc = self.q.get()
+
+ # Warn if the queue is too long.
+ # (Note: the other thread might have added entries to self.q
+ # after the .get() and before the .qsize().)
+ qsize = self.q.qsize()+1
+ if operation != OP_BOOT and qsize > BACKLOG_TOO_HIGH:
+ logging.warn('worker backlog is at %d', qsize)
+
try:
if operation == OP_UPDATE:
self._update(wc)
+ elif operation == OP_BOOT:
+ self._update(wc, boot=True)
elif operation == OP_CLEANUP:
self._cleanup(wc)
else:
@@ -228,7 +256,7 @@ class BackgroundWorker(threading.Thread)
self.q.put((operation, wc))
- def _update(self, wc):
+ def _update(self, wc, boot=False):
"Update the specified working copy."
# For giggles, let's clean up the working copy in case something
@@ -239,13 +267,15 @@ class BackgroundWorker(threading.Thread)
### we need to move some of these args into the config. these are
### still specific to the ASF setup.
- args = [self.svnbin, 'update',
+ args = [self.svnbin, 'switch',
'--quiet',
'--non-interactive',
'--trust-server-cert',
'--ignore-externals',
'--config-option',
'config:miscellany:use-commit-times=on',
+ '--',
+ wc.url,
wc.path]
subprocess.check_call(args, env=self.env)
@@ -253,6 +283,15 @@ class BackgroundWorker(threading.Thread)
info = svn_info(self.svnbin, self.env, wc.path)
logging.info("updated: %s now at r%s", wc.path, info['Revision'])
+ ## Run the hook
+ if self.hook:
+ hook_mode = ['post-update', 'boot'][boot]
+ logging.info('running hook: %s at revision %s due to %s',
+ wc.path, info['Revision'], hook_mode)
+ args = [self.hook, hook_mode,
+ wc.path, info['Revision'], wc.url]
+ subprocess.check_call(args, env=self.env)
+
def _cleanup(self, wc):
"Run a cleanup on the specified working copy."