You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@subversion.apache.org by pr...@apache.org on 2013/06/05 11:22:51 UTC
svn commit: r1489765 [22/22] - in /subversion/branches/verify-keep-going: ./
build/ build/ac-macros/ build/generator/ build/generator/templates/
contrib/hook-scripts/ contrib/server-side/fsfsfixer/
contrib/server-side/fsfsfixer/fixer/ notes/ subversion...
Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/client.py
URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/client.py?rev=1489765&r1=1489764&r2=1489765&view=diff
==============================================================================
--- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/client.py (original)
+++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/client.py Wed Jun 5 09:22:43 2013
@@ -62,7 +62,8 @@ class SvnpubsubClientException(Exception
class Client(asynchat.async_chat):
- def __init__(self, url, commit_callback, event_callback):
+ def __init__(self, url, commit_callback, event_callback,
+ metadata_callback = None):
asynchat.async_chat.__init__(self)
self.last_activity = time.time()
@@ -82,7 +83,8 @@ class Client(asynchat.async_chat):
self.event_callback = event_callback
- self.parser = JSONRecordHandler(commit_callback, event_callback)
+ self.parser = JSONRecordHandler(commit_callback, event_callback,
+ metadata_callback)
# Wait for the end of headers. Then we start parsing JSON.
self.set_terminator(b'\r\n\r\n')
@@ -94,7 +96,7 @@ class Client(asynchat.async_chat):
except:
self.handle_error()
return
-
+
self.push(('GET %s HTTP/1.0\r\n\r\n' % resource).encode('ascii'))
def handle_connect(self):
@@ -123,39 +125,53 @@ class Client(asynchat.async_chat):
self.last_activity = time.time()
if not self.skipping_headers:
- self.ibuffer.append(data)
+ self.ibuffer.append(data)
+
+
+class Notification(object):
+ def __init__(self, data):
+ self.__dict__.update(data)
+
+class Commit(Notification):
+ KIND = 'COMMIT'
+
+class Metadata(Notification):
+ KIND = 'METADATA'
class JSONRecordHandler:
- def __init__(self, commit_callback, event_callback):
+ def __init__(self, commit_callback, event_callback, metadata_callback):
self.commit_callback = commit_callback
self.event_callback = event_callback
+ self.metadata_callback = metadata_callback
+
+ EXPECTED_VERSION = 1
def feed(self, record):
obj = json.loads(record)
if 'svnpubsub' in obj:
actual_version = obj['svnpubsub'].get('version')
- EXPECTED_VERSION = 1
- if actual_version != EXPECTED_VERSION:
- raise SvnpubsubClientException("Unknown svnpubsub format: %r != %d"
- % (actual_format, expected_format))
+ if actual_version != self.EXPECTED_VERSION:
+ raise SvnpubsubClientException(
+ "Unknown svnpubsub format: %r != %d"
+ % (actual_version, self.EXPECTED_VERSION))
self.event_callback('version', obj['svnpubsub']['version'])
elif 'commit' in obj:
commit = Commit(obj['commit'])
self.commit_callback(commit)
elif 'stillalive' in obj:
self.event_callback('ping', obj['stillalive'])
-
-
-class Commit(object):
- def __init__(self, commit):
- self.__dict__.update(commit)
+ elif 'metadata' in obj and self.metadata_callback:
+ metadata = Metadata(obj['metadata'])
+ self.metadata_callback(metadata)
class MultiClient(object):
- def __init__(self, urls, commit_callback, event_callback):
+ def __init__(self, urls, commit_callback, event_callback,
+ metadata_callback = None):
self.commit_callback = commit_callback
self.event_callback = event_callback
+ self.metadata_callback = metadata_callback
# No target time, as no work to do
self.target_time = 0
@@ -185,9 +201,15 @@ class MultiClient(object):
def _add_channel(self, url):
# Simply instantiating the client will install it into the global map
# for processing in the main event loop.
- Client(url,
- functools.partial(self.commit_callback, url),
- functools.partial(self._reconnect, url))
+ if self.metadata_callback:
+ Client(url,
+ functools.partial(self.commit_callback, url),
+ functools.partial(self._reconnect, url),
+ functools.partial(self.metadata_callback, url))
+ else:
+ Client(url,
+ functools.partial(self.commit_callback, url),
+ functools.partial(self._reconnect, url))
def _check_stale(self):
now = time.time()
Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/server.py
URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/server.py?rev=1489765&r1=1489764&r2=1489765&view=diff
==============================================================================
--- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/server.py (original)
+++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnpubsub/server.py Wed Jun 5 09:22:43 2013
@@ -28,17 +28,24 @@
# Currently supports both XML and JSON serialization.
#
# Example Sub clients:
-# curl -sN http://127.0.0.1:2069/commits
-# curl -sN http://127.0.0.1:2069/commits/svn/*
-# curl -sN http://127.0.0.1:2069/commits/svn
-# curl -sN http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68
-# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68
-#
-# URL is built into 2 parts:
-# /commits/${optional_type}/${optional_repository}
-#
-# If the type is included in the URL, you will only get commits of that type.
-# The type can be * and then you will receive commits of any type.
+# curl -sN http://127.0.0.1:2069/commits
+# curl -sN 'http://127.0.0.1:2069/commits/svn/*'
+# curl -sN http://127.0.0.1:2069/commits/svn
+# curl -sN 'http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68'
+# curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68
+#
+# curl -sN http://127.0.0.1:2069/metadata
+# curl -sN 'http://127.0.0.1:2069/metadata/svn/*'
+# curl -sN http://127.0.0.1:2069/metadata/svn
+# curl -sN 'http://127.0.0.1:2069/metadata/*/13f79535-47bb-0310-9956-ffa450edef68'
+# curl -sN http://127.0.0.1:2069/metadata/svn/13f79535-47bb-0310-9956-ffa450edef68
+#
+# URLs are constructed from 3 parts:
+# /${notification}/${optional_type}/${optional_repository}
+#
+# Notifications can be sent for commits or metadata (e.g., revprop) changes.
+# If the type is included in the URL, you will only get notifications of that type.
+# The type can be * and then you will receive notifications of any type.
#
# If the repository is included in the URL, you will only receive
# messages about that repository. The repository can be * and then you
@@ -71,7 +78,7 @@ from twisted.python import log
import time
-class Commit:
+class Notification(object):
def __init__(self, r):
self.__dict__.update(r)
if not self.check_value('repository'):
@@ -82,11 +89,20 @@ class Commit:
raise ValueError('Invalid Format Value')
if not self.check_value('id'):
raise ValueError('Invalid ID Value')
-
+
def check_value(self, k):
return hasattr(self, k) and self.__dict__[k]
- def render_commit(self):
+ def render(self):
+ raise NotImplementedError
+
+ def render_log(self):
+ raise NotImplementedError
+
+class Commit(Notification):
+ KIND = 'COMMIT'
+
+ def render(self):
obj = {'commit': {}}
obj['commit'].update(self.__dict__)
return json.dumps(obj)
@@ -96,20 +112,32 @@ class Commit:
paths_changed = " %d paths changed" % len(self.changed)
except:
paths_changed = ""
- return "%s:%s repo '%s' id '%s'%s" % (self.type,
- self.format,
- self.repository,
- self.id,
- paths_changed)
+ return "commit %s:%s repo '%s' id '%s'%s" % (
+ self.type, self.format, self.repository, self.id,
+ paths_changed)
+
+class Metadata(Notification):
+ KIND = 'METADATA'
+
+ def render(self):
+ obj = {'metadata': {}}
+ obj['metadata'].update(self.__dict__)
+ return json.dumps(obj)
+
+ def render_log(self):
+ return "metadata %s:%s repo '%s' id '%s' revprop '%s'" % (
+ self.type, self.format, self.repository, self.id,
+ self.revprop['name'])
HEARTBEAT_TIME = 15
class Client(object):
- def __init__(self, pubsub, r, type, repository):
+ def __init__(self, pubsub, r, kind, type, repository):
self.pubsub = pubsub
r.notifyFinish().addErrback(self.finished)
self.r = r
+ self.kind = kind
self.type = type
self.repository = repository
self.alive = True
@@ -123,14 +151,17 @@ class Client(object):
except ValueError:
pass
- def interested_in(self, commit):
- if self.type and self.type != commit.type:
+ def interested_in(self, notification):
+ if self.kind != notification.KIND:
+ return False
+
+ if self.type and self.type != notification.type:
return False
- if self.repository and self.repository != commit.repository:
+ if self.repository and self.repository != notification.repository:
return False
- return True
+ return True
def notify(self, data):
self.write(data)
@@ -152,7 +183,10 @@ class Client(object):
self.r.write(str(input))
def write_start(self):
- self.r.setHeader('content-type', 'application/json')
+ # TODO: use application/x-* or vnd.* - see
+ # Message-ID: <CA...@mail.gmail.com>
+ # on May 2013
+ self.r.setHeader('content-type', 'application/octet-stream')
self.write('{"svnpubsub": {"version": 1}}\n\0')
def write_heartbeat(self):
@@ -163,6 +197,13 @@ class SvnPubSub(resource.Resource):
isLeaf = True
clients = []
+ __notification_uri_map = {'commits': Commit.KIND,
+ 'metadata': Metadata.KIND}
+
+ def __init__(self, notification_class):
+ resource.Resource.__init__(self)
+ self.__notification_class = notification_class
+
def cc(self):
return len(self.clients)
@@ -173,18 +214,23 @@ class SvnPubSub(resource.Resource):
log.msg("REQUEST: %s" % (request.uri))
request.setHeader('content-type', 'text/plain')
- repository = None
- type = None
+ repository = None
+ type = None
uri = request.uri.split('/')
uri_len = len(uri)
- if uri_len < 2 or uri_len > 4:
+ if uri_len < 2 or uri_len > 4:
request.setResponseCode(400)
return "Invalid path\n"
- if uri_len >= 3:
+ kind = self.__notification_uri_map.get(uri[1], None)
+ if kind is None:
+ request.setResponseCode(400)
+ return "Invalid path\n"
+
+ if uri_len >= 3:
type = uri[2]
-
+
if uri_len == 4:
repository = uri[3]
@@ -194,17 +240,18 @@ class SvnPubSub(resource.Resource):
if repository == '*':
repository = None
- c = Client(self, request, type, repository)
+ c = Client(self, request, kind, type, repository)
self.clients.append(c)
c.start()
return twisted.web.server.NOT_DONE_YET
- def notifyAll(self, commit):
- data = commit.render_commit()
+ def notifyAll(self, notification):
+ data = notification.render()
- log.msg("COMMIT: %s (%d clients)" % (commit.render_log(), self.cc()))
+ log.msg("%s: %s (%d clients)"
+ % (notification.KIND, notification.render_log(), self.cc()))
for client in self.clients:
- if client.interested_in(commit):
+ if client.interested_in(notification):
client.write_data(data)
def render_PUT(self, request):
@@ -217,19 +264,23 @@ class SvnPubSub(resource.Resource):
#import pdb;pdb.set_trace()
#print "input: %s" % (input)
try:
- c = json.loads(input)
- commit = Commit(c)
+ data = json.loads(input)
+ notification = self.__notification_class(data)
except ValueError as e:
request.setResponseCode(400)
- log.msg("COMMIT: failed due to: %s" % str(e))
- return str(e)
- self.notifyAll(commit)
+ errstr = str(e)
+ log.msg("%s: failed due to: %s" % (notification.KIND, errstr))
+ return errstr
+ self.notifyAll(notification)
return "Ok"
+
def svnpubsub_server():
root = resource.Resource()
- s = SvnPubSub()
- root.putChild("commits", s)
+ c = SvnPubSub(Commit)
+ m = SvnPubSub(Metadata)
+ root.putChild('commits', c)
+ root.putChild('metadata', m)
return server.Site(root)
if __name__ == "__main__":
Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svntweet.py
URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svntweet.py?rev=1489765&r1=1489764&r2=1489765&view=diff
==============================================================================
--- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svntweet.py (original)
+++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svntweet.py Wed Jun 5 09:22:43 2013
@@ -93,7 +93,7 @@ class JSONRecordHandler:
obj = json.loads(record)
if 'svnpubsub' in obj:
actual_version = obj['svnpubsub'].get('version')
- EXPECTED_VERSION = 1
+ EXPECTED_VERSION = 1
if actual_version != EXPECTED_VERSION:
raise ValueException("Unknown svnpubsub format: %r != %d"
% (actual_format, expected_format))
@@ -207,7 +207,7 @@ class BigDoEverythingClasss(object):
if path[0:1] == '/' and len(path) > 1:
path = path[1:]
- #TODO: allow URL to be configurable.
+ #TODO: allow URL to be configurable.
link = " - http://svn.apache.org/r%d" % (commit.id)
left -= len(link)
msg = "r%d in %s by %s: " % (commit.id, path, commit.committer)
Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py
URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py?rev=1489765&r1=1489764&r2=1489765&view=diff
==============================================================================
--- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py (original)
+++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/svnwcsub.py Wed Jun 5 09:22:43 2013
@@ -69,18 +69,7 @@ except ImportError:
import daemonize
import svnpubsub.client
-
-# check_output() is only available in Python 2.7. Allow us to run with
-# earlier versions
-try:
- check_output = subprocess.check_output
-except AttributeError:
- def check_output(args, env): # note: we only use these two args
- pipe = subprocess.Popen(args, stdout=subprocess.PIPE, env=env)
- output, _ = pipe.communicate()
- if pipe.returncode:
- raise subprocess.CalledProcessError(pipe.returncode, args)
- return output
+import svnpubsub.util
assert hasattr(subprocess, 'check_call')
def check_call(*args, **kwds):
@@ -103,7 +92,7 @@ def check_call(*args, **kwds):
def svn_info(svnbin, env, path):
"Run 'svn info' on the target path, returning a dict of info data."
args = [svnbin, "info", "--non-interactive", "--", path]
- output = check_output(args, env=env).strip()
+ output = svnpubsub.util.check_output(args, env=env).strip()
info = { }
for line in output.split('\n'):
idx = line.index(':')
@@ -452,7 +441,7 @@ def prepare_logging(logfile):
# Apply the handler to the root logger
root = logging.getLogger()
root.addHandler(handler)
-
+
### use logging.INFO for now. switch to cmdline option or a config?
root.setLevel(logging.INFO)
Modified: subversion/branches/verify-keep-going/tools/server-side/svnpubsub/watcher.py
URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/tools/server-side/svnpubsub/watcher.py?rev=1489765&r1=1489764&r2=1489765&view=diff
==============================================================================
--- subversion/branches/verify-keep-going/tools/server-side/svnpubsub/watcher.py (original)
+++ subversion/branches/verify-keep-going/tools/server-side/svnpubsub/watcher.py Wed Jun 5 09:22:43 2013
@@ -35,6 +35,9 @@ def _commit(url, commit):
print('COMMIT: from %s' % url)
pprint.pprint(vars(commit), indent=2)
+def _metadata(url, metadata):
+ print('METADATA: from %s' % url)
+ pprint.pprint(vars(metadata), indent=2)
def _event(url, event_name, event_arg):
if event_arg:
@@ -44,7 +47,7 @@ def _event(url, event_name, event_arg):
def main(urls):
- mc = svnpubsub.client.MultiClient(urls, _commit, _event)
+ mc = svnpubsub.client.MultiClient(urls, _commit, _event, _metadata)
mc.run_forever()
Modified: subversion/branches/verify-keep-going/win-tests.py
URL: http://svn.apache.org/viewvc/subversion/branches/verify-keep-going/win-tests.py?rev=1489765&r1=1489764&r2=1489765&view=diff
==============================================================================
--- subversion/branches/verify-keep-going/win-tests.py (original)
+++ subversion/branches/verify-keep-going/win-tests.py Wed Jun 5 09:22:43 2013
@@ -366,7 +366,7 @@ def locate_libs():
'mod_authz_svn', 'mod_authz_svn.so')
mod_dontdothat_path = os.path.join(abs_objdir, 'tools', 'server-side',
'mod_dontdothat', 'mod_dontdothat.so')
-
+
copy_changed_file(mod_dav_svn_path, abs_objdir)
copy_changed_file(mod_authz_svn_path, abs_objdir)
copy_changed_file(mod_dontdothat_path, abs_objdir)
@@ -396,7 +396,7 @@ class Svnserve:
self.path = os.path.join(abs_objdir,
'subversion', 'svnserve', self.name)
self.root = os.path.join(abs_builddir, CMDLINE_TEST_SCRIPT_NATIVE_PATH)
- self.proc_handle = None
+ self.proc = None
def __del__(self):
"Stop svnserve when the object is deleted"
@@ -414,26 +414,18 @@ class Svnserve:
else:
args = [self.name] + self.args
print('Starting %s %s' % (self.kind, self.name))
- try:
- import win32process
- import win32con
- args = ' '.join([self._quote(x) for x in args])
- self.proc_handle = (
- win32process.CreateProcess(self._quote(self.path), args,
- None, None, 0,
- win32con.CREATE_NEW_CONSOLE,
- None, None, win32process.STARTUPINFO()))[0]
- except ImportError:
- os.spawnv(os.P_NOWAIT, self.path, args)
+
+ self.proc = subprocess.Popen([self.path] + args[1:])
def stop(self):
- if self.proc_handle is not None:
+ if self.proc is not None:
try:
- import win32process
print('Stopping %s' % self.name)
- win32process.TerminateProcess(self.proc_handle, 0)
+ self.proc.poll();
+ if self.proc.returncode is None:
+ self.proc.kill();
return
- except ImportError:
+ except AttributeError:
pass
print('Svnserve.stop not implemented')
@@ -456,7 +448,7 @@ class Httpd:
self.bulkupdates_option = 'off'
self.service = service
- self.proc_handle = None
+ self.proc = None
self.path = os.path.join(self.httpd_dir, 'bin', self.name)
if short_circuit:
@@ -522,7 +514,7 @@ class Httpd:
fp.write('PidFile pid\n')
fp.write('ErrorLog log\n')
fp.write('Listen ' + str(self.httpd_port) + '\n')
-
+
if not no_log:
fp.write('LogFormat "%h %l %u %t \\"%r\\" %>s %b" common\n')
fp.write('Customlog log common\n')
@@ -549,7 +541,7 @@ class Httpd:
# Write LoadModule for Subversion modules
fp.write(self._svn_module('dav_svn_module', 'mod_dav_svn.so'))
fp.write(self._svn_module('authz_svn_module', 'mod_authz_svn.so'))
-
+
# And for mod_dontdothat
fp.write(self._svn_module('dontdothat_module', 'mod_dontdothat.so'))
@@ -597,7 +589,7 @@ class Httpd:
"Create empty mime.types file"
fp = open(self.httpd_mime_types, 'w')
fp.close()
-
+
def _create_dontdothat_file(self):
"Create empty mime.types file"
fp = open(self.dontdothat_file, 'w')
@@ -644,7 +636,7 @@ class Httpd:
' AuthUserFile ' + self._quote(self.httpd_users) + '\n' \
' Require valid-user\n' \
' DontDoThatConfigFile ' + self._quote(self.dontdothat_file) + '\n' \
- '</Location>\n'
+ '</Location>\n'
def start(self):
if self.service:
@@ -674,27 +666,18 @@ class Httpd:
"Start HTTPD as daemon"
print('Starting httpd as daemon')
print(self.httpd_args)
- try:
- import win32process
- import win32con
- args = ' '.join([self._quote(x) for x in self.httpd_args])
- self.proc_handle = (
- win32process.CreateProcess(self._quote(self.path), args,
- None, None, 0,
- win32con.CREATE_NEW_CONSOLE,
- None, None, win32process.STARTUPINFO()))[0]
- except ImportError:
- os.spawnv(os.P_NOWAIT, self.path, self.httpd_args)
+ self.proc = subprocess.Popen([self.path] + self.httpd_args[1:])
def _stop_daemon(self):
"Stop the HTTPD daemon"
- if self.proc_handle is not None:
+ if self.proc is not None:
try:
- import win32process
print('Stopping %s' % self.name)
- win32process.TerminateProcess(self.proc_handle, 0)
+ self.proc.poll();
+ if self.proc.returncode is None:
+ self.proc.kill();
return
- except ImportError:
+ except AttributeError:
pass
print('Httpd.stop_daemon not implemented')