You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whimsical.apache.org by Sam Ruby <ru...@apache.org> on 2016/02/19 00:53:30 UTC
[whimsy.git] [1/1] Commit a80395e: board agenda stress test and fixes
Commit a80395ec2971242e5c37b472c59a935e8c515d19:
board agenda stress test and fixes
Branch: refs/heads/master
Author: Sam Ruby <ru...@intertwingly.net>
Committer: Sam Ruby <ru...@intertwingly.net>
Pusher: rubys <ru...@apache.org>
------------------------------------------------------------
lib/whimsy/asf/ldap.rb | +++++++ --------
www/board/agenda/Rakefile | ++++
www/board/agenda/models/agenda.rb | +++++++++++ ---
www/board/agenda/test/stresstest.rb | +++++++++
www/board/agenda/views/actions/approve.json.rb | + -
------------------------------------------------------------
129 changes: 107 additions, 22 deletions.
------------------------------------------------------------
diff --git a/lib/whimsy/asf/ldap.rb b/lib/whimsy/asf/ldap.rb
index b5243c3..a884ca5 100644
--- a/lib/whimsy/asf/ldap.rb
+++ b/lib/whimsy/asf/ldap.rb
@@ -33,6 +33,7 @@
require 'weakref'
require 'net/http'
require 'base64'
+require 'thread'
module ASF
module LDAP
@@ -48,6 +49,9 @@ module LDAP
ldaps://ldap2-lw-eu.apache.org:636
)
+ CONNECT_LOCK = Mutex.new
+ HOST_QUEUE = Queue.new
+
# fetch configuration from apache/infrastructure-puppet
def self.puppet_config
return @puppet if @puppet
@@ -74,7 +78,10 @@ def self.puppet_ldapservers
def self.connect(test = true)
# Try each host at most once
hosts.length.times do
- host = next_host
+ # Ensure we use each host in turn
+ hosts.each {|host| HOST_QUEUE.push host} if HOST_QUEUE.empty?
+ host = HOST_QUEUE.shift
+
Wunderbar.info "Connecting to LDAP server: #{host}"
begin
@@ -105,10 +112,12 @@ def self.connect(test = true)
end
end
- # backwards compatibility for tools that called this interface
+ # public entry point for establishing a connection safely
def self.init_ldap(reset = false)
- @ldap = nil if reset
- @ldap ||= ASF::LDAP.connect(!reset)
+ ASF::LDAP::CONNECT_LOCK.synchronize do
+ @ldap = nil if reset
+ @ldap ||= ASF::LDAP.connect(!reset)
+ end
end
# determine where ldap.conf resides
@@ -612,20 +621,9 @@ def self.hosts
@hosts = hosts
end
- # Ensure we use each host in turn
- def self.next_host
- @he ||= hosts.cycle
- @he.next
- end
-
- # select LDAP host
- def self.host
- @host ||= hosts.sample
- end
-
# query and extract cert from openssl output
def self.extract_cert
- host = LDAP.host[%r{//(.*?)(/|$)}, 1]
+ host = hosts.sample[%r{//(.*?)(/|$)}, 1]
puts ['openssl', 's_client', '-connect', host, '-showcerts'].join(' ')
out, err, rc = Open3.capture3 'openssl', 's_client',
'-connect', host, '-showcerts'
diff --git a/www/board/agenda/Rakefile b/www/board/agenda/Rakefile
index 65332df..70ca673 100644
--- a/www/board/agenda/Rakefile
+++ b/www/board/agenda/Rakefile
@@ -82,6 +82,10 @@ task :work => ['test/work/board',
namespace :test do
task :setup => [:reset, :work]
task :server => 'server:test'
+
+ task :stress => :setup do
+ ruby 'test/stresstest.rb'
+ end
end
task :clobber do
diff --git a/www/board/agenda/models/agenda.rb b/www/board/agenda/models/agenda.rb
index a163827..527f8ca 100755
--- a/www/board/agenda/models/agenda.rb
+++ b/www/board/agenda/models/agenda.rb
@@ -56,7 +56,9 @@ def self.parse(file, mode)
end
# update agenda file in SVN
- def self.update(file, message, &block)
+ def self.update(file, message, retries=20, &block)
+ commit_rc = 999
+
# Create a temporary work directory
dir = Dir.mktmpdir
@@ -91,17 +93,23 @@ def self.update(file, message, &block)
# if the output differs, update and commit the file in question
if output != input
IO.write(path, output)
- _.system ['svn', 'commit', auth, path, '-m', message]
+ commit_rc = _.system ['svn', 'commit', auth, path, '-m', message]
@@seen[path] = File.mtime(path)
+ else
+ commit_rc = 0
end
end
# update the file in question; update output if mtime changed
# (it may not: during testing, commits are prevented)
path = File.join(FOUNDATION_BOARD, file)
- mtime = File.mtime(path) if output
- _.system ['svn', 'update', auth, path]
- output = IO.read(path) if mtime != File.mtime(path)
+ File.open(path, 'r') do |fh|
+ fh.flock(File::LOCK_EX)
+ _.system ['svn', 'cleanup', FOUNDATION_BOARD]
+ mtime = File.mtime(path) if output
+ _.system ['svn', 'update', auth, path]
+ output = IO.read(path) if mtime != File.mtime(path)
+ end
# reparse the file if the output changed
if output != baseline or mtime != File.mtime(path)
@@ -114,6 +122,15 @@ def self.update(file, message, &block)
ensure
FileUtils.rm_rf dir
+
+ unless commit_rc == 0
+ if retries > 0
+ sleep rand(41-retries*2)*0.1 if retries <= 20
+ update(file, message, retries-1, &block)
+ else
+ raise Exception.new("svn commit failed")
+ end
+ end
end
# listen for changes to agenda files
diff --git a/www/board/agenda/test/stresstest.rb b/www/board/agenda/test/stresstest.rb
new file mode 100644
index 0000000..216f945
--- /dev/null
+++ b/www/board/agenda/test/stresstest.rb
@@ -0,0 +1,66 @@
+require 'net/http'
+require 'whimsy/asf/rack'
+require 'thread'
+
+Dir.chdir File.expand_path('../..', __FILE__)
+system 'rake test:setup'
+
+# start server
+ENV['RACK_ENV'] = 'test'
+pid = fork { exec 'passenger start' }
+
+# make sure server is cleaned up on exit
+at_exit do
+ # shut down
+ Process.kill 'INT', pid
+ Process.waitpid pid
+end
+
+# wait for server to start
+10.times do |i|
+ begin
+ Net::HTTP.get_response(URI.parse("http://localhost:3000/"))
+ break
+ rescue Errno::ECONNREFUSED
+ end
+ sleep i*0.1
+end
+
+# everybody approve tomcat
+threads = ASF::Auth::DIRECTORS.map do |userid, initials|
+ Thread.new do
+ File.unlink "test/work/data/#{userid}.yml" rescue nil
+
+ # approve tomcat
+ uri = URI.parse("http://localhost:3000/json/approve")
+ http = Net::HTTP.new(uri.host, uri.port)
+ request = Net::HTTP::Post.new(uri.request_uri)
+ request.basic_auth(userid, "password")
+ request.set_form_data agenda: "board_agenda_2015_02_18.txt",
+ initials: initials, request: 'approve', attach: 'BX'
+ response = http.request(request)
+
+ # commit
+ uri = URI.parse("http://localhost:3000/json/commit")
+ http = Net::HTTP.new(uri.host, uri.port)
+ request = Net::HTTP::Post.new(uri.request_uri)
+ request.basic_auth(userid, "password")
+ request.set_form_data message: "approve tomcat", initials: initials
+ response = http.request(request)
+ # File.write("#{userid}.response", response.body.force_encoding('utf-8'))
+ end
+end
+
+# wait for threads to complete
+threads.each {|thread| thread.join}
+
+# verify approvals
+agenda = File.read('test/work/board/board_agenda_2015_02_18.txt')
+approvals = agenda[/Tomcat\.\s+approved: (.*)/, 1]
+print approvals.inspect + ' ...'
+if approvals.split(/,\s*/).sort == ASF::Auth::DIRECTORS.values.sort
+ puts 'success'
+else
+ puts 'failure'
+ exit
+end
diff --git a/www/board/agenda/views/actions/approve.json.rb b/www/board/agenda/views/actions/approve.json.rb
index 5644c9d..7e5d820 100644
--- a/www/board/agenda/views/actions/approve.json.rb
+++ b/www/board/agenda/views/actions/approve.json.rb
@@ -2,7 +2,7 @@
# Pre-app approval/unapproval/flagging/unflagging of an agenda item
Pending.update(env.user, @agenda) do |pending|
- agenda = Agenda.parse @agenda, :quick
+ agenda = Agenda.parse @agenda, :full
@initials ||= pending['initials']
approved = pending['approved']