You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ac...@apache.org on 2012/06/12 23:21:10 UTC

svn commit: r1349547 - in /qpid/trunk/qpid/cpp/src: qpid/ha/ tests/

Author: aconway
Date: Tue Jun 12 21:21:09 2012
New Revision: 1349547

URL: http://svn.apache.org/viewvc?rev=1349547&view=rev
Log:
QPID-3603: HA bug fixes around transition to ready status

- Simplify QueueGuard::firstSafe calculation.
- Fix error in setting initial queues - was not checking if replicated.
- Send ready status to backups. Tests hang, deadlock in opened()->RemoteBackup on primary?
- Fix deadlock in QueueGuard.
- Don't start guards inside ConnectionObserver::opened.

Modified:
    qpid/trunk/qpid/cpp/src/qpid/ha/BrokerInfo.cpp
    qpid/trunk/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp
    qpid/trunk/qpid/cpp/src/qpid/ha/HaBroker.cpp
    qpid/trunk/qpid/cpp/src/qpid/ha/Membership.cpp
    qpid/trunk/qpid/cpp/src/qpid/ha/Membership.h
    qpid/trunk/qpid/cpp/src/qpid/ha/Primary.cpp
    qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.cpp
    qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.h
    qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.cpp
    qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.h
    qpid/trunk/qpid/cpp/src/tests/brokertest.py
    qpid/trunk/qpid/cpp/src/tests/ha_tests.py

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/BrokerInfo.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/BrokerInfo.cpp?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/BrokerInfo.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/BrokerInfo.cpp Tue Jun 12 21:21:09 2012
@@ -97,7 +97,6 @@ void BrokerInfo::assign(const Variant::M
 std::ostream& operator<<(std::ostream& o, const BrokerInfo& b) {
     return o << b.getHostName() << ":" << b.getPort() << "("
              << printable(b.getStatus()) << ")";
-    // FIXME aconway 2012-06-06:  include << b.getSystemId()?
 }
 
 std::ostream& operator<<(std::ostream& o, const BrokerInfo::Set& infos) {

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/BrokerReplicator.cpp Tue Jun 12 21:21:09 2012
@@ -258,6 +258,7 @@ void BrokerReplicator::route(Deliverable
         if (headers->getAsString(QMF_CONTENT) == EVENT) {
             for (Variant::List::iterator i = list.begin(); i != list.end(); ++i) {
                 Variant::Map& map = i->asMap();
+                QPID_LOG(trace, "Broker replicator event: " << map);
                 Variant::Map& schema = map[SCHEMA_ID].asMap();
                 Variant::Map& values = map[VALUES].asMap();
                 if      (match<EventQueueDeclare>(schema)) doEventQueueDeclare(values);
@@ -271,6 +272,7 @@ void BrokerReplicator::route(Deliverable
         } else if (headers->getAsString(QMF_OPCODE) == QUERY_RESPONSE) {
             for (Variant::List::iterator i = list.begin(); i != list.end(); ++i) {
                 Variant::Map& map = i->asMap();
+                QPID_LOG(trace, "Broker replicator response: " << map);
                 string type = map[SCHEMA_ID].asMap()[CLASS_NAME].asString();
                 Variant::Map& values = map[VALUES].asMap();
                 framing::FieldTable args;

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/HaBroker.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/HaBroker.cpp?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/HaBroker.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/HaBroker.cpp Tue Jun 12 21:21:09 2012
@@ -268,6 +268,11 @@ void HaBroker::statusChanged(Mutex::Scop
 }
 
 void HaBroker::membershipUpdate(const Variant::List& brokers) {
+    // FIXME aconway 2012-06-12: nasty callback in callback, clean up.
+    BrokerInfo info;
+    if (getStatus() == CATCHUP && getMembership().get(systemId, info) && info.getStatus() == READY)
+        setStatus(READY);
+
     // No lock, only calls thread-safe objects.
     mgmtObject->set_members(brokers);
     broker.getManagementAgent()->raiseEvent(_qmf::EventMembersUpdate(brokers));

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/Membership.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/Membership.cpp?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/Membership.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/Membership.cpp Tue Jun 12 21:21:09 2012
@@ -43,7 +43,7 @@ void Membership::add(const BrokerInfo& b
 
 void Membership::remove(const types::Uuid& id) {
     sys::Mutex::ScopedLock l(lock);
-    BrokerMap::iterator i = brokers.find(id);
+    BrokerInfo::Map::iterator i = brokers.find(id);
     if (i != brokers.end()) {
         brokers.erase(i);
         update(l);
@@ -72,7 +72,7 @@ types::Variant::List Membership::asList(
 
 types::Variant::List Membership::asList(sys::Mutex::ScopedLock&) const {
     types::Variant::List list;
-    for (BrokerMap::const_iterator i = brokers.begin(); i != brokers.end(); ++i)
+    for (BrokerInfo::Map::const_iterator i = brokers.begin(); i != brokers.end(); ++i)
         list.push_back(i->second.asMap());
     return list;
 }
@@ -81,8 +81,6 @@ void Membership::update(sys::Mutex::Scop
     if (updateCallback) {
         types::Variant::List list = asList(l);
         sys::Mutex::ScopedUnlock u(lock);
-        // FIXME aconway 2012-06-06: messy: Make this a data object,
-        // move locking into HaBroker?
         updateCallback(list);
     }
     QPID_LOG(debug, " HA: Membership update: " << brokers);
@@ -95,11 +93,18 @@ BrokerInfo::Set Membership::otherBackups
 
 BrokerInfo::Set Membership::otherBackups(sys::Mutex::ScopedLock&) const {
     BrokerInfo::Set result;
-    for (BrokerMap::const_iterator i = brokers.begin(); i != brokers.end(); ++i)
+    for (BrokerInfo::Map::const_iterator i = brokers.begin(); i != brokers.end(); ++i)
         if (isBackup(i->second.getStatus()) && i->second.getSystemId() != self)
             result.insert(i->second);
     return result;
 }
 
+bool Membership::get(const types::Uuid& id, BrokerInfo& result) {
+    sys::Mutex::ScopedLock l(lock);
+    BrokerInfo::Map::iterator i = brokers.find(id);
+    if (i == brokers.end()) return false;
+    result = i->second;
+    return true;
+}
 
 }} // namespace qpid::ha

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/Membership.h
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/Membership.h?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/Membership.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/Membership.h Tue Jun 12 21:21:09 2012
@@ -55,16 +55,16 @@ class Membership
     void assign(const types::Variant::List&);
     types::Variant::List asList() const;
 
+    bool get(const types::Uuid& id, BrokerInfo& result);
+
   private:
-    typedef std::map<types::Uuid, BrokerInfo> BrokerMap;
     BrokerInfo::Set otherBackups(sys::Mutex::ScopedLock&) const;
     types::Variant::List asList(sys::Mutex::ScopedLock&) const;
     void update(sys::Mutex::ScopedLock&);
-    std::ostream& print(std::ostream& o, sys::Mutex::ScopedLock&) const;
 
     mutable sys::Mutex lock;
     types::Uuid self;
-    BrokerMap brokers;
+    BrokerInfo::Map brokers;
     UpdateCallback updateCallback;
 };
 

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/Primary.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/Primary.cpp?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/Primary.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/Primary.cpp Tue Jun 12 21:21:09 2012
@@ -74,8 +74,9 @@ Primary::Primary(HaBroker& hb, const Bro
     else {
         QPID_LOG(debug, logPrefix << "Expected backups: " << expect);
         for (BrokerInfo::Set::iterator i = expect.begin(); i != expect.end(); ++i) {
+            bool guard = true;  // Create queue guards immediately for expected backups.
             boost::shared_ptr<RemoteBackup> backup(
-                new RemoteBackup(*i, haBroker.getBroker(), haBroker.getReplicationTest()));
+                new RemoteBackup(*i, haBroker.getBroker(), haBroker.getReplicationTest(), guard));
             backups[i->getSystemId()] = backup;
             if (!backup->isReady()) initialBackups.insert(backup);
         }
@@ -107,6 +108,9 @@ void Primary::checkReady(Mutex::ScopedLo
 
 void Primary::checkReady(BackupMap::iterator i, Mutex::ScopedLock& l)  {
     if (i != backups.end() && i->second->isReady()) {
+        BrokerInfo info = i->second->getBrokerInfo();
+        info.setStatus(READY);
+        haBroker.getMembership().add(info);
         initialBackups.erase(i->second);
         checkReady(l);
     }
@@ -140,16 +144,17 @@ void Primary::opened(broker::Connection&
     Mutex::ScopedLock l(lock);
     BrokerInfo info;
     if (ha::ConnectionObserver::getBrokerInfo(connection, info)) {
-        haBroker.getMembership().add(info);
         BackupMap::iterator i = backups.find(info.getSystemId());
         if (i == backups.end()) {
             QPID_LOG(debug, logPrefix << "New backup connected: " << info);
+            bool guard = false; // Lazy-create queue guards, pre-creating them here could cause deadlock.
             backups[info.getSystemId()].reset(
-                new RemoteBackup(info, haBroker.getBroker(), haBroker.getReplicationTest()));
+                new RemoteBackup(info, haBroker.getBroker(), haBroker.getReplicationTest(), guard));
         }
         else {
             QPID_LOG(debug, logPrefix << "Known backup connected: " << info);
         }
+        haBroker.getMembership().add(info);
     }
 }
 

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.cpp?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.cpp Tue Jun 12 21:21:09 2012
@@ -49,21 +49,16 @@ class QueueGuard::QueueObserver : public
 
 
 QueueGuard::QueueGuard(broker::Queue& q, const BrokerInfo& info)
-    : queue(q), subscription(0), isFirstSet(false)
+    : queue(q), subscription(0)
 {
     std::ostringstream os;
     os << "HA primary guard " << queue.getName() << "@" << info.getLogId() << ": ";
     logPrefix = os.str();
     observer.reset(new QueueObserver(*this));
-    queue.addObserver(observer);         // We can now receive concurrent calls to dequeued
-    sys::Mutex::ScopedLock l(lock);
-    // Race between this thread and enqueued thread to set first safe position.
-    if (!isFirstSet) {
-        // Must set after addObserver so we don't miss any dequeues.
-        firstSafe = queue.getPosition()+1; // Next message will be safe.
-        isFirstSet = true;
-        QPID_LOG(debug, logPrefix << "First position (initial): " << firstSafe);
-    }
+    // Once we call addObserver we can get calls to enqueued and  dequeued
+    queue.addObserver(observer);
+    // Must set after addObserver so we don't miss any enqueues.
+    firstSafe = queue.getPosition()+1; // Next message will be safe.
 }
 
 QueueGuard::~QueueGuard() { cancel(); }
@@ -78,12 +73,6 @@ void QueueGuard::enqueued(const QueuedMe
         Mutex::ScopedLock l(lock);
         assert(!delayed.contains(qm.position));
         delayed += qm.position;
-        if (!isFirstSet) {
-            firstSafe = qm.position;
-            isFirstSet = true;
-            QPID_LOG(debug, logPrefix << "First position (enqueued): " << firstSafe);
-        }
-        assert(qm.position >= firstSafe);
     }
 }
 
@@ -132,7 +121,7 @@ void QueueGuard::complete(const QueuedMe
 }
 
 framing::SequenceNumber QueueGuard::getFirstSafe() {
-    // No lock, first is immutable.
+    // No lock, firstSafe is immutable.
     return firstSafe;
 }
 

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.h
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.h?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/QueueGuard.h Tue Jun 12 21:21:09 2012
@@ -92,8 +92,6 @@ class QueueGuard {
     framing::SequenceSet delayed;
     ReplicatingSubscription* subscription;
     boost::shared_ptr<QueueObserver> observer;
-
-    bool isFirstSet;
     framing::SequenceNumber firstSafe; // Immutable
 };
 }} // namespace qpid::ha

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.cpp?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.cpp Tue Jun 12 21:21:09 2012
@@ -31,10 +31,12 @@ namespace ha {
 using sys::Mutex;
 
 RemoteBackup::RemoteBackup(
-    const BrokerInfo& info, broker::Broker& broker, ReplicationTest rt) :
-    logPrefix("HA primary, backup to "+info.getLogId()+": "), brokerInfo(info), replicationTest(rt)
+    const BrokerInfo& info, broker::Broker& broker, ReplicationTest rt, bool cg) :
+    logPrefix("HA primary, backup to "+info.getLogId()+": "), brokerInfo(info), replicationTest(rt),
+    createGuards(cg)
 {
     QPID_LOG(debug, logPrefix << "Guarding queues for backup broker.");
+    // FIXME aconway 2012-06-12: potential deadlocks, this is called inside ConnectionObserver::opened.
     broker.getQueues().eachQueue(boost::bind(&RemoteBackup::initialQueue, this, _1));
 }
 
@@ -48,29 +50,43 @@ bool RemoteBackup::isReady() {
 }
 
 void RemoteBackup::initialQueue(const QueuePtr& q) {
-    initialQueues.insert(q);
+    if (replicationTest.isReplicated(ALL, *q)) initialQueues.insert(q);
     queueCreate(q);
 }
 
 RemoteBackup::GuardPtr RemoteBackup::guard(const QueuePtr& q) {
+    if (!createGuards) return RemoteBackup::GuardPtr();
     GuardMap::iterator i = guards.find(q);
     if (i == guards.end()) {
         assert(0);
-        throw Exception(logPrefix+": Guard cannot find queue guard: "+q->getName());
+        throw Exception(logPrefix+": Cannot find queue guard: "+q->getName());
     }
     GuardPtr guard = i->second;
     guards.erase(i);
     return guard;
 }
 
+namespace {
+typedef std::set<boost::shared_ptr<broker::Queue> > QS;
+struct QueueSetPrinter {
+    const QS& qs;
+    QueueSetPrinter(const QS& q) : qs(q) {}
+};
+std::ostream& operator<<(std::ostream& o, const QueueSetPrinter& qp) {
+    for (QS::const_iterator i = qp.qs.begin(); i != qp.qs.end(); ++i)
+        o << (*i)->getName() << " ";
+    return o;
+}
+}
+
 void RemoteBackup::ready(const QueuePtr& q) {
-    QPID_LOG(debug, logPrefix << "Queue ready: " << q->getName());
     initialQueues.erase(q);
+    QPID_LOG(debug, logPrefix << "Queue ready: " << q->getName() << " remaining unready: " << QueueSetPrinter(initialQueues));
     if (isReady()) QPID_LOG(debug, logPrefix << "All queues ready");
 }
 
 void RemoteBackup::queueCreate(const QueuePtr& q) {
-    if (replicationTest.isReplicated(ALL, *q))
+    if (createGuards && replicationTest.isReplicated(ALL, *q))
         guards[q].reset(new QueueGuard(*q, brokerInfo));
 }
 

Modified: qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.h
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.h?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/ha/RemoteBackup.h Tue Jun 12 21:21:09 2012
@@ -51,7 +51,7 @@ class RemoteBackup
     typedef boost::shared_ptr<QueueGuard> GuardPtr;
     typedef boost::shared_ptr<broker::Queue> QueuePtr;
 
-    RemoteBackup(const BrokerInfo& info, broker::Broker&, ReplicationTest rt);
+    RemoteBackup(const BrokerInfo& info, broker::Broker&, ReplicationTest rt, bool createGuards);
     ~RemoteBackup();
 
     /** Return guard associated with a queue. Used to create ReplicatingSubscription. */
@@ -67,6 +67,7 @@ class RemoteBackup
     /**@return true when all initial queues for this backup are ready */
     bool isReady();
 
+    BrokerInfo getBrokerInfo() const { return brokerInfo; }
   private:
     typedef std::map<QueuePtr, GuardPtr> GuardMap;
     typedef std::set<QueuePtr> QueueSet;
@@ -76,6 +77,7 @@ class RemoteBackup
     ReplicationTest replicationTest;
     GuardMap guards;
     QueueSet initialQueues;
+    bool createGuards;
 
     void initialQueue(const QueuePtr&);
 };

Modified: qpid/trunk/qpid/cpp/src/tests/brokertest.py
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/tests/brokertest.py?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/tests/brokertest.py (original)
+++ qpid/trunk/qpid/cpp/src/tests/brokertest.py Tue Jun 12 21:21:09 2012
@@ -243,7 +243,7 @@ class Broker(Popen):
     _broker_count = 0
     _log_count = 0
 
-    def __str__(self): return "Broker<%s %s>"%(self.name, self.pname)
+    def __str__(self): return "Broker<%s %s :%d>"%(self.name, self.pname, self.port())
 
     def find_log(self):
         self.log = "%03d:%s.log" % (Broker._log_count, self.name)

Modified: qpid/trunk/qpid/cpp/src/tests/ha_tests.py
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/tests/ha_tests.py?rev=1349547&r1=1349546&r2=1349547&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/tests/ha_tests.py (original)
+++ qpid/trunk/qpid/cpp/src/tests/ha_tests.py Tue Jun 12 21:21:09 2012
@@ -65,6 +65,8 @@ class HaBroker(Broker):
         self.qpid_ha_script=import_script(self.qpid_ha_path)
         self._agent = None
 
+    def __str__(self): return Broker.__str__(self)
+
     def qpid_ha(self, args): self.qpid_ha_script.main(["", "-b", self.host_port()]+args)
 
     def promote(self): self.qpid_ha(["promote"])
@@ -78,7 +80,7 @@ class HaBroker(Broker):
     def ha_status(self): return self.agent().getHaBroker().status
 
     def wait_status(self, status):
-        assert retry(lambda: self.ha_status() == status), "%r != %r"%(self.ha_status(), status)
+        assert retry(lambda: self.ha_status() == status), "%s, %r != %r"%(self, self.ha_status(), status)
 
     # FIXME aconway 2012-05-01: do direct python call to qpid-config code.
     def qpid_config(self, args):
@@ -738,10 +740,12 @@ class LongTests(BrokerTest):
                 for r in receivers: r.receiver.assert_running()
                 n = receivers[0].received
                 # FIXME aconway 2012-05-01: don't kill primary till it's active
-                # otherwise we can lose messages. When we implement non-promotion
-                # of catchup brokers we can make this stronger: wait only for
-                # there to be at least one ready backup.
+                # and backups are ready, otherwise we can lose messages. When we
+                # implement non-promotion of catchup brokers we can make this
+                # stronger: wait only for there to be at least one ready backup.
                 brokers[i%3].wait_status("active")
+                brokers[(i+1)%3].wait_status("ready")
+                brokers[(i+2)%3].wait_status("ready")
                 brokers.bounce(i%3)
                 i += 1
                 def enough():        # Verify we're still running



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org