You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ac...@apache.org on 2009/04/29 22:29:54 UTC

svn commit: r769914 - in /qpid/trunk/qpid/cpp/src: ./ qpid/cluster/

Author: aconway
Date: Wed Apr 29 20:29:53 2009
New Revision: 769914

URL: http://svn.apache.org/viewvc?rev=769914&view=rev
Log:
Improved & simplified cluster output algorithm.

Removed:
    qpid/trunk/qpid/cpp/src/qpid/cluster/WriteEstimate.cpp
    qpid/trunk/qpid/cpp/src/qpid/cluster/WriteEstimate.h
Modified:
    qpid/trunk/qpid/cpp/src/cluster.mk
    qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp
    qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.h
    qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterPlugin.cpp
    qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterSettings.h
    qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.cpp
    qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.h
    qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.cpp
    qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.h

Modified: qpid/trunk/qpid/cpp/src/cluster.mk
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/cluster.mk?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/cluster.mk (original)
+++ qpid/trunk/qpid/cpp/src/cluster.mk Wed Apr 29 20:29:53 2009
@@ -78,8 +78,6 @@
   qpid/cluster/PollerDispatch.h			\
   qpid/cluster/ProxyInputHandler.h		\
   qpid/cluster/Quorum.h				\
-  qpid/cluster/WriteEstimate.cpp		\
-  qpid/cluster/WriteEstimate.h			\
   qpid/cluster/types.h 
 
 cluster_la_LIBADD=  -lcpg $(libcman) libqpidbroker.la libqpidclient.la

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.cpp Wed Apr 29 20:29:53 2009
@@ -96,8 +96,6 @@
     name(settings.name),
     myUrl(settings.url.empty() ? Url() : Url(settings.url)),
     self(cpg.self()),
-    readMax(settings.readMax),
-    writeEstimate(settings.writeEstimate),
     expiryPolicy(new ExpiryPolicy(mcast, self, broker.getTimer())),
     mcast(cpg, poller, boost::bind(&Cluster::leave, this)),
     dispatcher(cpg, poller, boost::bind(&Cluster::leave, this)),

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.h
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.h?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/Cluster.h Wed Apr 29 20:29:53 2009
@@ -101,8 +101,7 @@
 
     void checkQuorum();
 
-    size_t getReadMax() { return readMax; }
-    size_t getWriteEstimate() { return writeEstimate; }
+    const ClusterSettings& getSettings() const { return settings; }
 
     void deliverFrame(const EventFrame&);
 
@@ -192,7 +191,7 @@
     void updateOutDone(Lock&);
 
     // Immutable members set on construction, never changed.
-    ClusterSettings settings;
+    const ClusterSettings settings;
     broker::Broker& broker;
     qmf::org::apache::qpid::cluster::Cluster* mgmtObject; // mgnt owns lifecycle
     boost::shared_ptr<sys::Poller> poller;
@@ -200,8 +199,6 @@
     const std::string name;
     Url myUrl;
     const MemberId self;
-    const size_t readMax;
-    const size_t writeEstimate;
     framing::Uuid clusterId;
     NoOpConnectionOutputHandler shadowOut;
     qpid::management::ManagementAgent* mAgent;

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterPlugin.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterPlugin.cpp?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterPlugin.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterPlugin.cpp Wed Apr 29 20:29:53 2009
@@ -74,7 +74,9 @@
             ("cluster-read-max", optValue(settings.readMax,"N"),
              "Experimental: Limit per-client-connection queue of read buffers. 0=no limit.")
             ("cluster-write-estimate", optValue(settings.writeEstimate, "Kb"),
-             "Experimental: initial estimate for connection write rate per multicast cycle")
+             "Experimental: initial estimate for write rate per multicast cycle")
+            ("cluster-write-min", optValue(settings.writeMin, "Kb"),
+             "Experimental: minimum estimate for write rate per multicast cycle")
             ;
     }
 };

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterSettings.h
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterSettings.h?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterSettings.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/ClusterSettings.h Wed Apr 29 20:29:53 2009
@@ -32,10 +32,10 @@
     std::string name;
     std::string url;
     bool quorum;
-    size_t readMax, writeEstimate;
+    size_t readMax, writeEstimate, writeMin;
     std::string username, password, mechanism;
 
-    ClusterSettings() : quorum(false), readMax(10), writeEstimate(64) {}
+    ClusterSettings() : quorum(false), readMax(10), writeEstimate(1), writeMin(1) {}
   
     Url getUrl(uint16_t port) const {
         if (url.empty()) return Url::getIpAddressesUrl(port);

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.cpp?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.cpp Wed Apr 29 20:29:53 2009
@@ -89,7 +89,7 @@
     if (isLocalClient()) {  
         connection.setClusterOrderOutput(mcastFrameHandler); // Actively send cluster-order frames from local node
         cluster.addLocalConnection(this);
-        giveReadCredit(cluster.getReadMax());
+        giveReadCredit(cluster.getSettings().readMax);
     }
     else {                                                  // Shadow or catch-up connection
         connection.setClusterOrderOutput(nullFrameHandler); // Passive, discard cluster-order frames
@@ -100,7 +100,7 @@
 }
 
 void Connection::giveReadCredit(int credit) {
-    if (cluster.getReadMax() && credit) 
+    if (cluster.getSettings().readMax && credit) 
         output.giveReadCredit(credit);
 }
 

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.h
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.h?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/Connection.h Wed Apr 29 20:29:53 2009
@@ -23,7 +23,6 @@
  */
 
 #include "types.h"
-#include "WriteEstimate.h"
 #include "OutputInterceptor.h"
 #include "EventFrame.h"
 #include "McastFrameHandler.h"
@@ -178,7 +177,6 @@
     Cluster& cluster;
     ConnectionId self;
     bool catchUp;
-    WriteEstimate writeEstimate;
     OutputInterceptor output;
     framing::FrameDecoder localDecoder;
     broker::Connection connection;

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.cpp
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.cpp?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.cpp (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.cpp Wed Apr 29 20:29:53 2009
@@ -36,13 +36,16 @@
 
 OutputInterceptor::OutputInterceptor(Connection& p, sys::ConnectionOutputHandler& h)
     : parent(p), closing(false), next(&h), sent(),
-      writeEstimate(p.getCluster().getWriteEstimate()),
+      estimate(p.getCluster().getSettings().writeEstimate*1024),
+      minimum(p.getCluster().getSettings().writeMin*1024),
       moreOutput(), doingOutput()
 {}
 
 void OutputInterceptor::send(framing::AMQFrame& f) {
     parent.getCluster().checkQuorum();
     {
+        // FIXME aconway 2009-04-28: locking around next-> may be redundant
+        // with the fixes to read-credit in the IO layer. Review.
         sys::Mutex::ScopedLock l(lock);
         next->send(f);
     }
@@ -58,7 +61,7 @@
     else if (!closing) {        // Don't send do ouput after output stopped.
         QPID_LOG(trace,  parent << " activateOutput - sending doOutput");
         moreOutput = true;
-        sendDoOutput();
+        sendDoOutput(estimate);
     }
 }
 
@@ -77,36 +80,35 @@
 // 
 void OutputInterceptor::deliverDoOutput(size_t requested) {
     size_t buf = getBuffered();
-    if (parent.isLocal())
-        writeEstimate.delivered(requested, sent, buf); // Update the estimate.
-
-    // Run the real doOutput() till we have added the requested data or there's nothing to output.
+    if (parent.isLocal()) {  // Adjust estimate for next sendDoOutput 
+        sent = sent > buf ? sent - buf : 0; // Buffered data was not sent.
+        if (buf > 0)          // Wrote to capacity, move estimate towards sent.
+            estimate = (estimate + sent) /2;  
+        else if (sent >= estimate) // Last estimate was too small, increase it.
+            estimate *= 2;  
+        if (estimate < minimum) estimate = minimum;
+    }
+    // Run the real doOutput() till we have added the requested data
+    // or there's nothing to output. Record how much we send.
     sent = 0;
     do {
         moreOutput = parent.getBrokerConnection().doOutput();
     } while (sent < requested && moreOutput);
-    sent += buf;                // Include buffered data in the sent total.
-    QPID_LOG(trace, parent << " delivereDoOutput: requested=" << requested << " sent=" << sent << " more=" << moreOutput);
-    if (parent.isLocal() && moreOutput)  {
-        QPID_LOG(trace,  parent << " deliverDoOutput - sending doOutput, more output available.");
-        sendDoOutput();
-    }
-    else
+    sent += buf;                // Include data previously in the buffer
+
+    if (parent.isLocal()) {
+        // Send the next doOutput request
         doingOutput = false;
+        sendDoOutput(estimate); // FIXME aconway 2009-04-28: account for data in buffer?
+    }
 }
 
 // Send a doOutput request if one is not already in flight.
-void OutputInterceptor::sendDoOutput() {
-    if (!parent.isLocal()) return;
+void OutputInterceptor::sendDoOutput(size_t request) {
+    if (!parent.isLocal() || doingOutput || !moreOutput) return;
     doingOutput = true;
-    size_t request = writeEstimate.sending(getBuffered());
-    
-    // Note we may send 0 size request if there's more than 2*estimate in the buffer.
-    // Send it anyway to keep the doOutput chain going until we are sure there's no more output
-    // (in deliverDoOutput)
-    //
     parent.getCluster().getMulticast().mcastControl(
-        ClusterConnectionDeliverDoOutputBody(ProtocolVersion(), request), parent.getId());
+        ClusterConnectionDeliverDoOutputBody(ProtocolVersion(), estimate), parent.getId());
     QPID_LOG(trace, parent << "Send doOutput request for " << request);
 }
 

Modified: qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.h
URL: http://svn.apache.org/viewvc/qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.h?rev=769914&r1=769913&r2=769914&view=diff
==============================================================================
--- qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.h (original)
+++ qpid/trunk/qpid/cpp/src/qpid/cluster/OutputInterceptor.h Wed Apr 29 20:29:53 2009
@@ -22,9 +22,9 @@
  *
  */
 
-#include "WriteEstimate.h"
 #include "NoOpConnectionOutputHandler.h"
 #include "qpid/sys/ConnectionOutputHandler.h"
+#include "qpid/sys/Mutex.h"
 #include "qpid/broker/ConnectionFactory.h"
 #include <boost/function.hpp>
 
@@ -60,14 +60,14 @@
   private:
     typedef sys::Mutex::ScopedLock Locker;
 
-    void sendDoOutput();
+    void sendDoOutput(size_t);
 
     mutable sys::Mutex lock;
     bool closing;
     sys::ConnectionOutputHandler* next;
     size_t sent;
-    size_t lastDoOutput;
-    WriteEstimate writeEstimate;
+    size_t estimate;
+    size_t minimum;
     bool moreOutput;
     bool doingOutput;
     static NoOpConnectionOutputHandler discardHandler;



---------------------------------------------------------------------
Apache Qpid - AMQP Messaging Implementation
Project:      http://qpid.apache.org
Use/Interact: mailto:commits-subscribe@qpid.apache.org