You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by tr...@apache.org on 2017/10/18 14:43:24 UTC
[02/15] qpid-dispatch git commit: DISPATCH-829 - Added labels and
debug logs to diagnose delivery ref-count issues.
DISPATCH-829 - Added labels and debug logs to diagnose delivery ref-count issues.
Project: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/repo
Commit: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/commit/9871fdbf
Tree: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/tree/9871fdbf
Diff: http://git-wip-us.apache.org/repos/asf/qpid-dispatch/diff/9871fdbf
Branch: refs/heads/master
Commit: 9871fdbfccb749c410a9f5d2ed9784e542026233
Parents: 653ed7c
Author: Ted Ross <tr...@redhat.com>
Authored: Mon Oct 16 13:10:43 2017 -0400
Committer: Ted Ross <tr...@redhat.com>
Committed: Wed Oct 18 08:11:40 2017 -0400
----------------------------------------------------------------------
include/qpid/dispatch/router_core.h | 4 +--
src/router_core/connections.c | 18 +++++-----
src/router_core/forwarder.c | 4 +--
src/router_core/router_core_private.h | 2 +-
src/router_core/transfer.c | 55 ++++++++++++++++--------------
src/router_node.c | 14 ++++----
6 files changed, 51 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9871fdbf/include/qpid/dispatch/router_core.h
----------------------------------------------------------------------
diff --git a/include/qpid/dispatch/router_core.h b/include/qpid/dispatch/router_core.h
index dfa1c9f..2b18da9 100644
--- a/include/qpid/dispatch/router_core.h
+++ b/include/qpid/dispatch/router_core.h
@@ -606,8 +606,8 @@ void qdr_delivery_update_disposition(qdr_core_t *core, qdr_delivery_t *delivery,
void qdr_delivery_set_context(qdr_delivery_t *delivery, void *context);
void *qdr_delivery_get_context(qdr_delivery_t *delivery);
-void qdr_delivery_incref(qdr_delivery_t *delivery);
-void qdr_delivery_decref(qdr_core_t *core, qdr_delivery_t *delivery);
+void qdr_delivery_incref(qdr_delivery_t *delivery, const char *label);
+void qdr_delivery_decref(qdr_core_t *core, qdr_delivery_t *delivery, const char *label);
void qdr_delivery_tag(const qdr_delivery_t *delivery, const char **tag, int *length);
qd_message_t *qdr_delivery_message(const qdr_delivery_t *delivery);
qdr_error_t *qdr_delivery_error(const qdr_delivery_t *delivery);
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9871fdbf/src/router_core/connections.c
----------------------------------------------------------------------
diff --git a/src/router_core/connections.c b/src/router_core/connections.c
index c239507..8ed317c 100644
--- a/src/router_core/connections.c
+++ b/src/router_core/connections.c
@@ -314,7 +314,7 @@ int qdr_connection_process(qdr_connection_t *conn)
qdr_delivery_ref_t *dref = DEQ_HEAD(updated_deliveries);
while (dref) {
core->delivery_update_handler(core->user_context, dref->dlv, dref->dlv->disposition, dref->dlv->settled);
- qdr_delivery_decref(core, dref->dlv);
+ qdr_delivery_decref(core, dref->dlv, "qdr_connection_process - remove from updated list");
qdr_del_delivery_ref(&updated_deliveries, dref);
dref = DEQ_HEAD(updated_deliveries);
event_count++;
@@ -644,12 +644,12 @@ static void qdr_link_cleanup_deliveries_CT(qdr_core_t *core, qdr_connection_t *c
//
if (!ref->dlv->cleared_proton_ref) {
ref->dlv->cleared_proton_ref = true;
- qdr_delivery_decref_CT(core, ref->dlv);
+ qdr_delivery_decref_CT(core, ref->dlv, "qdr_link_cleanup_deliveries_CT - lost proton ref (1)");
}
//
// Now our reference
//
- qdr_delivery_decref_CT(core, ref->dlv);
+ qdr_delivery_decref_CT(core, ref->dlv, "qdr_link_cleanup_deliveries_CT - remove from updated list");
qdr_del_delivery_ref(&updated_deliveries, ref);
ref = DEQ_HEAD(updated_deliveries);
}
@@ -676,12 +676,12 @@ static void qdr_link_cleanup_deliveries_CT(qdr_core_t *core, qdr_connection_t *c
//
if (link->link_direction == QD_INCOMING && !dlv->settled && !dlv->cleared_proton_ref) {
dlv->cleared_proton_ref = true;
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_cleanup_deliveries_CT - lost proton ref (2) ");
}
//
// Now the undelivered-list reference
//
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_cleanup_deliveries_CT - remove from undelivered list");
dlv = DEQ_HEAD(undelivered);
}
@@ -721,12 +721,12 @@ static void qdr_link_cleanup_deliveries_CT(qdr_core_t *core, qdr_connection_t *c
//
if (!dlv->cleared_proton_ref) {
dlv->cleared_proton_ref = true;
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_cleanup_deliveries_CT - lost proton ref (3)");
}
//
// Now the unsettled-list reference
//
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_cleanup_deliveries_CT - remove from unsettled list");
dlv = DEQ_HEAD(unsettled);
}
@@ -754,11 +754,11 @@ static void qdr_link_cleanup_deliveries_CT(qdr_core_t *core, qdr_connection_t *c
//
if (!dlv->cleared_proton_ref) {
dlv->cleared_proton_ref = true;
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_cleanup_deliveries_CT - lost proton ref (4)");
}
// This decref is for the removing the delivery from the settled list
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_cleanup_deliveries_CT - remove from settled list");
dlv = DEQ_HEAD(settled);
}
}
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9871fdbf/src/router_core/forwarder.c
----------------------------------------------------------------------
diff --git a/src/router_core/forwarder.c b/src/router_core/forwarder.c
index 232868d..0fff358 100644
--- a/src/router_core/forwarder.c
+++ b/src/router_core/forwarder.c
@@ -162,7 +162,7 @@ static void qdr_forward_drop_presettled_CT_LH(qdr_core_t *core, qdr_link_t *link
free_qdr_link_work_t(dlv->link_work);
dlv->link_work = 0;
}
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_forward_drop_presettled_CT_LH - remove from link-work list");
}
dlv = next;
}
@@ -185,7 +185,7 @@ void qdr_forward_deliver_CT(qdr_core_t *core, qdr_link_t *out_link, qdr_delivery
out_dlv->where = QDR_DELIVERY_IN_UNDELIVERED;
// This incref is for putting the delivery in the undelivered list
- qdr_delivery_incref(out_dlv);
+ qdr_delivery_incref(out_dlv, "qdr_forward_deliver_CT - add to undelivered list");
//
// We must put a work item on the link's work list to represent this pending delivery.
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9871fdbf/src/router_core/router_core_private.h
----------------------------------------------------------------------
diff --git a/src/router_core/router_core_private.h b/src/router_core/router_core_private.h
index 33340d2..f2a1909 100644
--- a/src/router_core/router_core_private.h
+++ b/src/router_core/router_core_private.h
@@ -717,7 +717,7 @@ void qdr_delivery_push_CT(qdr_core_t *core, qdr_delivery_t *dlv);
void qdr_delivery_release_CT(qdr_core_t *core, qdr_delivery_t *delivery);
void qdr_delivery_failed_CT(qdr_core_t *core, qdr_delivery_t *delivery);
bool qdr_delivery_settled_CT(qdr_core_t *core, qdr_delivery_t *delivery);
-void qdr_delivery_decref_CT(qdr_core_t *core, qdr_delivery_t *delivery);
+void qdr_delivery_decref_CT(qdr_core_t *core, qdr_delivery_t *delivery, const char *label);
void qdr_forward_on_message_CT(qdr_core_t *core, qdr_subscription_t *sub, qdr_link_t *link, qd_message_t *msg);
/**
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9871fdbf/src/router_core/transfer.c
----------------------------------------------------------------------
diff --git a/src/router_core/transfer.c b/src/router_core/transfer.c
index 68ff6b2..d7f310a 100644
--- a/src/router_core/transfer.c
+++ b/src/router_core/transfer.c
@@ -20,6 +20,7 @@
#include "router_core_private.h"
#include <qpid/dispatch/amqp.h>
#include <stdio.h>
+#include <inttypes.h>
static void qdr_link_deliver_CT(qdr_core_t *core, qdr_action_t *action, bool discard);
static void qdr_link_flow_CT(qdr_core_t *core, qdr_action_t *action, bool discard);
@@ -121,7 +122,7 @@ qdr_delivery_t *qdr_deliver_continue(qdr_delivery_t *in_dlv)
action->args.connection.delivery = in_dlv;
// This incref is for the action reference
- qdr_delivery_incref(in_dlv);
+ qdr_delivery_incref(in_dlv, "qdr_deliver_continue - add to action list");
qdr_action_enqueue(in_dlv->link->core, action);
return in_dlv;
}
@@ -159,7 +160,7 @@ int qdr_link_process_deliveries(qdr_core_t *core, qdr_link_t *link, int credit)
dlv->where = QDR_DELIVERY_NOWHERE;
// This decref is for removing this settled delivery from the undelivered list
- qdr_delivery_decref(core, dlv);
+ qdr_delivery_decref(core, dlv, "qdr_link_process_deliveries - remove from undelivered list");
} else {
DEQ_INSERT_TAIL(link->unsettled, dlv);
@@ -267,7 +268,7 @@ void qdr_delivery_update_disposition(qdr_core_t *core, qdr_delivery_t *delivery,
// the given ref rather than increment a new one.
//
if (!ref_given)
- qdr_delivery_incref(delivery);
+ qdr_delivery_incref(delivery, "qdr_delivery_update_disposition - add to action");
qdr_action_enqueue(core, action);
}
@@ -331,9 +332,12 @@ bool qdr_delivery_receive_complete(const qdr_delivery_t *delivery)
}
-void qdr_delivery_incref(qdr_delivery_t *delivery)
+void qdr_delivery_incref(qdr_delivery_t *delivery, const char *label)
{
- sys_atomic_inc(&delivery->ref_count);
+ uint32_t rc = sys_atomic_inc(&delivery->ref_count);
+ if (delivery && delivery->link)
+ qd_log(delivery->link->core->log, QD_LOG_DEBUG,
+ "qdr_delivery_incref: dlv:%lx rc:%"PRIu32" label:%s", (long) delivery, rc + 1, label);
}
void qdr_delivery_set_aborted(const qdr_delivery_t *delivery, bool aborted)
@@ -352,10 +356,11 @@ bool qdr_delivery_is_aborted(const qdr_delivery_t *delivery)
}
-void qdr_delivery_decref(qdr_core_t *core, qdr_delivery_t *delivery)
+void qdr_delivery_decref(qdr_core_t *core, qdr_delivery_t *delivery, const char *label)
{
uint32_t ref_count = sys_atomic_dec(&delivery->ref_count);
assert(ref_count > 0);
+ qd_log(core->log, QD_LOG_DEBUG, "qdr_delivery_decref: dlv:%lx rc:%"PRIu32" label:%s", (long) delivery, ref_count - 1, label);
if (ref_count == 1) {
//
@@ -408,7 +413,7 @@ void qdr_delivery_release_CT(qdr_core_t *core, qdr_delivery_t *dlv)
// Remove the unsettled reference
//
if (moved)
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_delivery_release_CT - remove from unsettled list");
}
@@ -427,7 +432,7 @@ void qdr_delivery_failed_CT(qdr_core_t *core, qdr_delivery_t *dlv)
// Remove the unsettled reference
//
if (moved)
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_delivery_failed_CT - remove from unsettled list");
}
@@ -572,8 +577,8 @@ void qdr_delivery_link_peers_CT(qdr_delivery_t *in_dlv, qdr_delivery_t *out_dlv)
out_dlv->peer = in_dlv;
- qdr_delivery_incref(out_dlv);
- qdr_delivery_incref(in_dlv);
+ qdr_delivery_incref(out_dlv, "qdr_delivery_link_peers_CT - linked to peer (1)");
+ qdr_delivery_incref(in_dlv, "qdr_delivery_link_peers_CT - linked to peer (2)");
}
@@ -592,8 +597,8 @@ void qdr_delivery_unlink_peers_CT(qdr_core_t *core, qdr_delivery_t *dlv, qdr_del
assert(dlv->peer == peer);
dlv->peer = 0;
peer->peer = 0;
- qdr_delivery_decref_CT(core, dlv);
- qdr_delivery_decref_CT(core, peer);
+ qdr_delivery_decref_CT(core, dlv, "qdr_delivery_unlink_peers_CT - unlinked from peer (1)");
+ qdr_delivery_decref_CT(core, peer, "qdr_delivery_unlink_peers_CT - unlinked from peer (2)");
}
else {
//
@@ -607,9 +612,9 @@ void qdr_delivery_unlink_peers_CT(qdr_core_t *core, qdr_delivery_t *dlv, qdr_del
qdr_del_delivery_ref(&dlv->peers, peer_ref);
if (peer->peer == dlv) {
peer->peer = 0;
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_delivery_unlink_peers_CT - unlinked from peer (3)");
}
- qdr_delivery_decref_CT(core, peer);
+ qdr_delivery_decref_CT(core, peer, "qdr_delivery_unlink_peers_CT - unlinked from peer (4)");
break;
}
peer_ref = DEQ_NEXT(peer_ref);
@@ -659,11 +664,11 @@ qdr_delivery_t *qdr_delivery_next_peer_CT(qdr_delivery_t *dlv)
}
-void qdr_delivery_decref_CT(qdr_core_t *core, qdr_delivery_t *dlv)
+void qdr_delivery_decref_CT(qdr_core_t *core, qdr_delivery_t *dlv, const char *label)
{
uint32_t ref_count = sys_atomic_dec(&dlv->ref_count);
-
assert(ref_count > 0);
+ qd_log(core->log, QD_LOG_DEBUG, "qdr_delivery_decref_CT: dlv:%lx rc:%"PRIu32" label:%s", (long) dlv, ref_count - 1, label);
if (ref_count == 1)
qdr_delete_delivery_internal_CT(core, dlv);
@@ -821,7 +826,7 @@ static void qdr_link_forward_CT(qdr_core_t *core, qdr_link_t *link, qdr_delivery
//
// Decrementing the delivery ref count for the action
//
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_forward_CT - removed from action (1)");
qdr_link_issue_credit_CT(core, link, 1, false);
} else if (fanout > 0) {
if (dlv->settled || qdr_is_addr_treatment_multicast(addr)) {
@@ -834,7 +839,7 @@ static void qdr_link_forward_CT(qdr_core_t *core, qdr_link_t *link, qdr_delivery
//
// This decref is for the action ref
//
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_forward_CT - removed from action (2)");
}
else {
//
@@ -910,7 +915,7 @@ static void qdr_link_deliver_CT(qdr_core_t *core, qdr_action_t *action, bool dis
// If the delivery is settled, decrement the ref_count on the delivery.
// This count was the owned-by-action count.
//
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_link_deliver_CT - removed from action");
}
return;
}
@@ -1025,15 +1030,15 @@ static void qdr_update_delivery_CT(qdr_core_t *core, qdr_action_t *action, bool
//
// Release the action reference, possibly freeing the delivery
//
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_update_delivery_CT - remove from action");
//
// Release the unsettled references if the deliveries were moved
//
if (dlv_moved)
- qdr_delivery_decref_CT(core, dlv);
+ qdr_delivery_decref_CT(core, dlv, "qdr_update_delivery_CT - removed from unsettled (1)");
if (peer_moved)
- qdr_delivery_decref_CT(core, peer);
+ qdr_delivery_decref_CT(core, peer, "qdr_update_delivery_CT - removed from unsettled (2)");
if (error_unassigned)
qdr_error_free(error);
}
@@ -1086,7 +1091,7 @@ static void qdr_deliver_continue_CT(qdr_core_t *core, qdr_action_t *action, bool
qdr_delivery_t *in_dlv = action->args.connection.delivery;
// This decref is for the action reference
- qdr_delivery_decref_CT(core, in_dlv);
+ qdr_delivery_decref_CT(core, in_dlv, "qdr_deliver_continue_CT - remove from action");
//
// If it is already in the undelivered list or it has no peers, don't try to deliver this again.
@@ -1132,7 +1137,7 @@ static void qdr_deliver_continue_CT(qdr_core_t *core, qdr_action_t *action, bool
// Remove the delivery from the settled list and decref the in_dlv.
in_dlv->where = QDR_DELIVERY_NOWHERE;
DEQ_REMOVE(in_dlv->link->settled, in_dlv);
- qdr_delivery_decref_CT(core, in_dlv); // This decref is for removing the delivery from the settled list.
+ qdr_delivery_decref_CT(core, in_dlv, "qdr_deliver_continue_CT - remove from settled list");
}
}
}
@@ -1240,7 +1245,7 @@ void qdr_delivery_push_CT(qdr_core_t *core, qdr_delivery_t *dlv)
sys_mutex_lock(link->conn->work_lock);
if (dlv->where != QDR_DELIVERY_IN_UNDELIVERED) {
- qdr_delivery_incref(dlv);
+ qdr_delivery_incref(dlv, "qdr_delivery_push_CT - add to updated list");
qdr_add_delivery_ref_CT(&link->updated_deliveries, dlv);
qdr_add_link_ref(&link->conn->links_with_work, link, QDR_LINK_LIST_CLASS_WORK);
activate = true;
http://git-wip-us.apache.org/repos/asf/qpid-dispatch/blob/9871fdbf/src/router_node.c
----------------------------------------------------------------------
diff --git a/src/router_node.c b/src/router_node.c
index aadc312..e6267b4 100644
--- a/src/router_node.c
+++ b/src/router_node.c
@@ -269,7 +269,7 @@ static void AMQP_rx_handler(void* context, qd_link_t *link)
//
if (pn_delivery_settled(pnd) && receive_complete) {
pn_delivery_settle(pnd);
- qdr_delivery_decref(router->router_core, delivery);
+ qdr_delivery_decref(router->router_core, delivery, "AMQP_rx_handler - link route settled delivery");
}
}
else {
@@ -282,7 +282,7 @@ static void AMQP_rx_handler(void* context, qd_link_t *link)
pn_disposition_data(pn_delivery_remote(pnd)));
pn_delivery_set_context(pnd, delivery);
qdr_delivery_set_context(delivery, pnd);
- qdr_delivery_incref(delivery);
+ qdr_delivery_incref(delivery, "AMQP_rx_handler - link-route add to proton delivery");
}
return;
@@ -346,7 +346,7 @@ static void AMQP_rx_handler(void* context, qd_link_t *link)
if (receive_complete) {
if (pn_delivery_settled(pnd) || pn_delivery_aborted(pnd)) {
pn_delivery_settle(pnd);
- qdr_delivery_decref(router->router_core, delivery);
+ qdr_delivery_decref(router->router_core, delivery, "AMQP_rx_handler - message-route settled");
}
}
@@ -474,7 +474,7 @@ static void AMQP_rx_handler(void* context, qd_link_t *link)
// If this delivery is unsettled or if this is a settled multi-frame large message, set the context
pn_delivery_set_context(pnd, delivery);
qdr_delivery_set_context(delivery, pnd);
- qdr_delivery_incref(delivery);
+ qdr_delivery_incref(delivery, "AMQP_rx_handler - message-route add proton delivery");
} else {
//
// If there is no delivery, the message is now and will always be unroutable because there is no address.
@@ -1278,7 +1278,7 @@ static void CORE_link_deliver(void *context, qdr_link_t *link, qdr_delivery_t *d
pn_delivery_set_context(pdlv, dlv);
qdr_delivery_set_context(dlv, pdlv);
qdr_delivery_set_set_proton_ref(dlv, true);
- qdr_delivery_incref(dlv);
+ qdr_delivery_incref(dlv, "CORE_link_deliver - add to proton delivery");
}
}
@@ -1325,7 +1325,7 @@ static void CORE_link_deliver(void *context, qdr_link_t *link, qdr_delivery_t *d
qdr_delivery_set_cleared_proton_ref(dlv, true);
if (qdr_delivery_get_set_proton_ref(dlv)) {
- qdr_delivery_decref(router->router_core, dlv);
+ qdr_delivery_decref(router->router_core, dlv, "CORE_link_deliver - get_set_proton_ref");
}
}
@@ -1389,7 +1389,7 @@ static void CORE_delivery_update(void *context, qdr_delivery_t *dlv, uint64_t di
pn_delivery_set_context(pnd, 0);
pn_delivery_settle(pnd);
qdr_delivery_set_cleared_proton_ref(dlv, true);
- qdr_delivery_decref(router->router_core, dlv);
+ qdr_delivery_decref(router->router_core, dlv, "CORE_delivery_update - remove proton delivery");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org