You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by ja...@apache.org on 2015/07/07 02:07:45 UTC
[1/2] trafficserver git commit: Add LastFailure to hostdb stats
endpoint
Repository: trafficserver
Updated Branches:
refs/heads/master 4282d3353 -> 6e35feb88
Add LastFailure to hostdb stats endpoint
This way you can have insight into hostdb's view of last_failures.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/6e35feb8
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/6e35feb8
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/6e35feb8
Branch: refs/heads/master
Commit: 6e35feb88d4c6a0835b7c2013a315240b1adf23b
Parents: 02678a1
Author: Thomas Jackson <ja...@apache.org>
Authored: Thu Jul 2 18:24:28 2015 -0700
Committer: Thomas Jackson <ja...@apache.org>
Committed: Mon Jul 6 17:07:22 2015 -0700
----------------------------------------------------------------------
iocore/hostdb/HostDB.cc | 41 +++++++++++++++++++++++++++++-----
iocore/hostdb/I_HostDBProcessor.h | 2 ++
iocore/hostdb/P_HostDBProcessor.h | 2 +-
3 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/6e35feb8/iocore/hostdb/HostDB.cc
----------------------------------------------------------------------
diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc
index 838d34f..037c437 100644
--- a/iocore/hostdb/HostDB.cc
+++ b/iocore/hostdb/HostDB.cc
@@ -870,6 +870,25 @@ HostDBProcessor::getbyname_re(Continuation *cont, const char *ahostname, int len
return getby(cont, ahostname, len, 0, force_dns, opt.host_res_style, opt.timeout);
}
+Action *
+HostDBProcessor::getbynameport_re(Continuation *cont, const char *ahostname, int len, Options const &opt)
+{
+ bool force_dns = false;
+ EThread *thread = this_ethread();
+ ProxyMutex *mutex = thread->mutex;
+
+ if (opt.flags & HOSTDB_FORCE_DNS_ALWAYS)
+ force_dns = true;
+ else if (opt.flags & HOSTDB_FORCE_DNS_RELOAD) {
+ force_dns = (hostdb_re_dns_on_reload ? true : false);
+ if (force_dns)
+ HOSTDB_INCREMENT_DYN_STAT(hostdb_re_dns_on_reload_stat);
+ }
+ sockaddr sa;
+ ats_ip4_set(&sa, NULL, htons(opt.port));
+ return getby(cont, ahostname, len, &sa, force_dns, opt.host_res_style, opt.timeout);
+}
+
/* Support SRV records */
Action *
@@ -2267,6 +2286,7 @@ struct ShowHostDB;
typedef int (ShowHostDB::*ShowHostDBEventHandler)(int event, Event *data);
struct ShowHostDB : public ShowCont {
char *name;
+ uint16_t port;
IpEndpoint ip;
bool force;
@@ -2294,10 +2314,12 @@ struct ShowHostDB : public ShowCont {
showLookup(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
{
SET_HANDLER(&ShowHostDB::showLookupDone);
- if (name)
- hostDBProcessor.getbyname_re(this, name, 0,
- HostDBProcessor::Options().setFlags(force ? HostDBProcessor::HOSTDB_FORCE_DNS_ALWAYS : 0));
- else
+ if (name) {
+ HostDBProcessor::Options opts;
+ opts.port = port;
+ opts.flags = HostDBProcessor::HOSTDB_DO_NOT_FORCE_DNS;
+ hostDBProcessor.getbynameport_re(this, name, strlen(name), opts);
+ } else
hostDBProcessor.getbyaddr_re(this, &ip.sa);
return EVENT_CONT;
}
@@ -2312,6 +2334,7 @@ struct ShowHostDB : public ShowCont {
r->reverse_dns ? "Reverse DNS" : "DNS"));
CHECK_SHOW(show("<tr><td>%s</td><td>%u</td></tr>\n", "App1", r->app.allotment.application1));
CHECK_SHOW(show("<tr><td>%s</td><td>%u</td></tr>\n", "App2", r->app.allotment.application2));
+ CHECK_SHOW(show("<tr><td>%s</td><td>%u</td></tr>\n", "LastFailure", r->app.http_data.last_failure));
if (!rr) {
CHECK_SHOW(show("<tr><td>%s</td><td>%s</td></tr>\n", "Stale", r->is_ip_stale() ? "Yes" : "No"));
CHECK_SHOW(show("<tr><td>%s</td><td>%s</td></tr>\n", "Timed-Out", r->is_ip_timeout() ? "Yes" : "No"));
@@ -2403,8 +2426,16 @@ register_ShowHostDB(Continuation *c, HTTPHdr *h)
char *gn = NULL;
if (s->sarg)
gn = (char *)memchr(s->sarg, '=', strlen(s->sarg));
- if (gn)
+ if (gn) {
s->name = gn + 1;
+ char *pos = strstr(s->name, "%3A");
+ if (pos != NULL) {
+ s->port = atoi(pos + 3);
+ *pos = '\0'; // Null terminate name
+ } else {
+ s->port = 0;
+ }
+ }
SET_CONTINUATION_HANDLER(s, &ShowHostDB::showLookup);
}
this_ethread()->schedule_imm(s);
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/6e35feb8/iocore/hostdb/I_HostDBProcessor.h
----------------------------------------------------------------------
diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h
index 43051bf..ea46839 100644
--- a/iocore/hostdb/I_HostDBProcessor.h
+++ b/iocore/hostdb/I_HostDBProcessor.h
@@ -452,6 +452,8 @@ struct HostDBProcessor : public Processor {
inkcoreapi Action *getbyname_re(Continuation *cont, const char *hostname, int len, Options const &opt = DEFAULT_OPTIONS);
+ Action *getbynameport_re(Continuation *cont, const char *hostname, int len, Options const &opt = DEFAULT_OPTIONS);
+
Action *getSRVbyname_imm(Continuation *cont, process_srv_info_pfn process_srv_info, const char *hostname, int len,
Options const &opt = DEFAULT_OPTIONS);
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/6e35feb8/iocore/hostdb/P_HostDBProcessor.h
----------------------------------------------------------------------
diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h
index cc3200f..99f9426 100644
--- a/iocore/hostdb/P_HostDBProcessor.h
+++ b/iocore/hostdb/P_HostDBProcessor.h
@@ -191,7 +191,7 @@ extern RecRawStatBlock *hostdb_rsb;
struct CmpConstBuffferCaseInsensitive {
- bool operator()(ts::ConstBuffer a, ts::ConstBuffer b) { return ptr_len_casecmp(a._ptr, a._size, b._ptr, b._size) < 0;}
+ bool operator()(ts::ConstBuffer a, ts::ConstBuffer b) { return ptr_len_casecmp(a._ptr, a._size, b._ptr, b._size) < 0; }
};
// Our own typedef for the host file mapping
[2/2] trafficserver git commit: Maintain and use a mapping of
hostname -> IP to implement host file support
Posted by ja...@apache.org.
Maintain and use a mapping of hostname -> IP to implement host file support
This allows the background thread to populate a map of overrides, which we simply inject in the lookup method-- in place of calling to DNS.
Fixes: TS-3725
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/02678a1f
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/02678a1f
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/02678a1f
Branch: refs/heads/master
Commit: 02678a1f8c9abc92985fbf6d7bb8a4b02467ba6b
Parents: 4282d33
Author: Thomas Jackson <ja...@apache.org>
Authored: Wed Jul 1 16:52:55 2015 -0700
Committer: Thomas Jackson <ja...@apache.org>
Committed: Mon Jul 6 17:07:22 2015 -0700
----------------------------------------------------------------------
iocore/hostdb/HostDB.cc | 263 +++++----------------------------
iocore/hostdb/P_HostDBProcessor.h | 15 ++
2 files changed, 51 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/02678a1f/iocore/hostdb/HostDB.cc
----------------------------------------------------------------------
diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc
index c9c54d9..838d34f 100644
--- a/iocore/hostdb/HostDB.cc
+++ b/iocore/hostdb/HostDB.cc
@@ -238,6 +238,7 @@ HostDBCache::HostDBCache()
max_hits = (1 << HOST_DB_HITS_BITS) - 1;
version.ink_major = HOST_DB_CACHE_MAJOR_VERSION;
version.ink_minor = HOST_DB_CACHE_MINOR_VERSION;
+ hosts_file_ptr = new RefCountedHostsFileMap();
}
@@ -659,6 +660,21 @@ db_mark_for(IpAddr const &ip)
HostDBInfo *
probe(ProxyMutex *mutex, HostDBMD5 const &md5, bool ignore_timeout)
{
+ // If we have an entry in our hosts file, we don't need to bother with DNS
+ Ptr<RefCountedHostsFileMap> current_host_file_map = hostDB.hosts_file_ptr;
+
+ ts::ConstBuffer hname(md5.host_name, md5.host_len);
+ HostsFileMap::iterator find_result = current_host_file_map->hosts_file_map.find(hname);
+ if (find_result != current_host_file_map->hosts_file_map.end()) {
+ // TODO: Something to make this not local :/
+ static HostDBInfo r;
+ r.round_robin = false;
+ r.reverse_dns = false;
+ r.is_srv = false;
+ ats_ip_set(r.ip(), (*find_result).second);
+ return &r;
+ }
+
ink_assert(this_ethread() == hostDB.lock_for_bucket((int)(fold_md5(md5.hash) % hostDB.buckets))->thread_holding);
if (hostdb_enable) {
uint64_t folded_md5 = fold_md5(md5.hash);
@@ -2527,19 +2543,6 @@ struct HostDBFileContinuation : public Continuation {
HostDBFileContinuation() : Continuation(0) {}
- int insertEvent(int event, void *data);
- int removeEvent(int event, void *data);
-
- /// Set the current entry in the HostDB.
- HostDBInfo *setDBEntry();
- /// Create and schedule an update continuation.
- static void scheduleUpdate(int idx, ///< Pair index to process.
- Keys *keys = 0 ///< Key table if any.
- );
- /// Create and schedule a remove continuation.
- static void scheduleRemove(int idx, ///< Index of current new key to check against.
- Keys *keys ///< new valid keys
- );
/// Finish update
static void finish(Keys *keys ///< Valid keys from update.
);
@@ -2562,21 +2565,9 @@ HostDBFileContinuation::destroy()
// proceeding at a time in any case so we might as well make these
// globals.
int HostDBFileUpdateActive = 0;
-// Contents of the host file. We keep this around because other data
-// points in to it (to minimize allocations).
-ats_scoped_str HostFileText;
-// Accumulated pairs of <IP address, host>
-std::vector<HostFilePair> HostFilePairs;
-// Entries from last update.
-HostDBFileContinuation::Keys HostFileKeys;
-/// Ordering operator for HostFilePair.
-/// We want to group first by name and then by address family.
-bool
-CmpHostFilePair(HostFilePair const &lhs, HostFilePair const &rhs)
-{
- int zret = strcasecmp(lhs.name, rhs.name);
- return zret < 0 || (0 == zret && lhs.ip < rhs.ip);
-}
+// map of hostname -> IpAddr
+Ptr<RefCountedHostsFileMap> parsed_hosts_file_ptr(new RefCountedHostsFileMap());
+
// Actual ordering doesn't matter as long as it's consistent.
bool
CmpMD5(INK_MD5 const &lhs, INK_MD5 const &rhs)
@@ -2584,185 +2575,6 @@ CmpMD5(INK_MD5 const &lhs, INK_MD5 const &rhs)
return lhs[0] < rhs[0] || (lhs[0] == rhs[0] && lhs[1] < rhs[1]);
}
-/// Finish current update.
-void
-HostDBFileContinuation::finish(Keys *keys)
-{
- HostFilePairs.clear();
- HostFileKeys.clear();
- if (keys) {
- std::swap(*keys, HostFileKeys); // put the new keys in place and dump the previous.
- delete keys;
- }
-
- HostFileText = 0;
- HostDBFileUpdateActive = 0;
-}
-
-HostDBInfo *
-HostDBFileContinuation::setDBEntry()
-{
- HostDBInfo *r;
- uint64_t folded_md5 = fold_md5(md5);
-
- // remove the old one to prevent buildup
- if (0 != (r = hostDB.lookup_block(folded_md5, hostDB.levels))) {
- hostDB.delete_block(r);
- Debug("hostdb", "Update host file entry %s - %" PRIx64 ".%" PRIx64, name, md5[0], md5[1]);
- } else {
- Debug("hostdb", "Add host file entry %s - %" PRIx64 ".%" PRIx64, name, md5[0], md5[1]);
- }
-
- r = hostDB.insert_block(folded_md5, NULL, 0);
- r->md5_high = md5[1];
- r->ip_timeout_interval = 0; // special value - no timeout.
- r->ip_timestamp = hostdb_current_interval;
- keys->push_back(md5);
- return r;
-}
-
-int
-HostDBFileContinuation::insertEvent(int, void *)
-{
- int n = HostFilePairs.size();
- HostFilePair const &first = HostFilePairs[idx];
- int last = idx + 1;
- HostDBInfo *r = 0;
-
- ink_assert(idx < n);
-
- // Get the set of addresses for this name.
- while (last < n && 0 == strcasecmp(first.name, HostFilePairs[last].name) && first.ip.family() == HostFilePairs[last].ip.family())
- ++last;
- // Address set is now (idx, last].
-
- int k = last - idx; // # of addrs for this name
- ink_assert(k > 0);
-
- r = this->setDBEntry();
- r->reverse_dns = false;
- r->is_srv = false;
- r->data.ip.assign(HostFilePairs[idx].ip);
- if (k > 1) {
- // multiple entries, need round robin
- int s = HostDBRoundRobin::size(k, false);
- HostDBRoundRobin *rr_data = static_cast<HostDBRoundRobin *>(hostDB.alloc(&r->app.rr.offset, s));
- if (rr_data) {
- int dst = 0; // index of destination RR item.
- for (int src = idx; src < last; ++src, ++dst) {
- HostDBInfo &item = rr_data->info[dst];
- item.data.ip.assign(HostFilePairs[src].ip);
- item.full = 1;
- item.round_robin = false;
- item.reverse_dns = false;
- item.is_srv = false;
- item.md5_high = r->md5_high;
- item.md5_low = r->md5_low;
- item.md5_low_low = r->md5_low_low;
- }
- r->round_robin = true;
- rr_data->good = k;
- rr_data->current = 0;
- rr_data->rrcount = dst;
- }
- } else {
- r->round_robin = false;
- }
-
- if (last < n) { // more entries to process
- // We create a new continuation rather than using this one to
- // avoid mutex issues - it is critical that the continuation use
- // the bucket mutex to guarantee a data lock for update. It's
- // unclear if changing the mutex to this continuation will do that
- // properly. Because we use a class allocator we should achieve
- // efficient allocations quickly even for large host files.
- this->scheduleUpdate(last, keys);
- } else {
- std::sort(keys->begin(), keys->end(), &CmpMD5);
- // keys->qsort(&CmpMD5);
- // Switch to removing dead entries.
- this->scheduleRemove(keys->size() - 1, keys);
- }
- // This continuation is done.
- this->destroy();
- return EVENT_DONE;
-}
-
-int
-HostDBFileContinuation::removeEvent(int, void *)
-{
- HostDBInfo *r;
- uint64_t folded_md5 = fold_md5(md5);
- Debug("hostdb", "Remove host file entry %" PRIx64 ".%" PRIx64, md5[0], md5[1]);
- if (0 != (r = hostDB.lookup_block(folded_md5, 3)))
- hostDB.delete_block(r);
- this->scheduleRemove(idx, keys);
- this->destroy();
- return EVENT_DONE;
-}
-
-void
-HostDBFileContinuation::scheduleUpdate(int idx, Keys *keys)
-{
- HostDBFileContinuation *c = hostDBFileContAllocator.alloc();
- HostFilePair &pair = HostFilePairs[idx];
- HostDBMD5 md5;
-
- md5.set_host(pair.name, strlen(pair.name));
- md5.db_mark = db_mark_for(pair.ip);
- md5.refresh();
-
- SET_CONTINUATION_HANDLER(c, &HostDBFileContinuation::insertEvent);
- c->idx = idx;
- c->name = pair.name;
- c->keys = keys ? keys : new Keys;
- c->md5 = md5.hash;
- c->mutex = hostDB.lock_for_bucket(fold_md5(md5.hash) % hostDB.buckets);
- eventProcessor.schedule_imm(c, ET_CALL, EVENT_IMMEDIATE, 0);
-}
-
-void
-HostDBFileContinuation::scheduleRemove(int idx, Keys *keys)
-{
- bool targetp = false; // Have a valid target?
- INK_MD5 md5;
-
- // See if we have a target.
- if (HostFileKeys.size() > 0) {
- md5 = HostFileKeys.back();
- HostFileKeys.pop_back();
- targetp = true;
- // Move backwards through the valid items and the previous
- // keys, removing if we have a prev key that's bigger than
- // the current valid key (which means it's not currently valid).
- while (idx >= 0) {
- if (CmpMD5((*keys)[idx], md5)) { // prev is not valid, remove
- break;
- } else if (CmpMD5(md5, (*keys)[idx])) {
- --idx; // prev is smaller, skip current and keep checking
- } else if (!HostFileKeys.empty()) {
- md5 = HostFileKeys.back();
- HostFileKeys.pop_back(); // match, move to next potential target
- --idx;
- } else { // ran out of things to remove.
- targetp = false;
- break;
- }
- }
- }
-
- if (targetp) {
- HostDBFileContinuation *c = hostDBFileContAllocator.alloc();
- SET_CONTINUATION_HANDLER(c, &HostDBFileContinuation::removeEvent);
- c->md5 = md5;
- c->idx = idx;
- c->keys = keys;
- c->mutex = hostDB.lock_for_bucket(fold_md5(c->md5) % hostDB.buckets);
- eventProcessor.schedule_imm(c, ET_CALL, EVENT_IMMEDIATE, 0);
- } else {
- self::finish(keys);
- }
-}
void
ParseHostLine(char *l)
@@ -2771,11 +2583,14 @@ ParseHostLine(char *l)
int n_elts = elts.Initialize(l, SHARE_TOKS);
// Elements should be the address then a list of host names.
// Don't use RecHttpLoadIp because the address *must* be literal.
- HostFilePair item;
- if (n_elts > 1 && 0 == item.ip.load(elts[0]) && !item.ip.isLoopback()) {
+ IpAddr ip;
+ if (n_elts > 1 && 0 == ip.load(elts[0]) && !ip.isLoopback()) {
for (int i = 1; i < n_elts; ++i) {
- item.name = elts[i];
- HostFilePairs.push_back(item);
+ ts::ConstBuffer name(elts[i], strlen(elts[i]));
+ // If we don't have an entry already (host files only support single IPs for a given name)
+ if (parsed_hosts_file_ptr->hosts_file_map.find(name) == parsed_hosts_file_ptr->hosts_file_map.end()) {
+ parsed_hosts_file_ptr->hosts_file_map[name] = ip;
+ }
}
}
}
@@ -2798,13 +2613,14 @@ ParseHostFile(char const *path)
if (0 == fstat(fd, &info)) {
// +1 in case no terminating newline
int64_t size = info.st_size + 1;
- HostFileText = static_cast<char *>(ats_malloc(size));
- if (HostFileText) {
- char *base = HostFileText;
+
+ parsed_hosts_file_ptr->HostFileText = static_cast<char *>(ats_malloc(size));
+ if (parsed_hosts_file_ptr->HostFileText) {
+ char *base = parsed_hosts_file_ptr->HostFileText;
char *limit;
- size = read(fd, HostFileText, info.st_size);
- limit = HostFileText + size;
+ size = read(fd, parsed_hosts_file_ptr->HostFileText, info.st_size);
+ limit = parsed_hosts_file_ptr->HostFileText + size;
*limit = 0;
// We need to get a list of all name/addr pairs so that we can
@@ -2833,15 +2649,8 @@ ParseHostFile(char const *path)
}
}
- if (!HostFilePairs.empty()) {
- // Need to sort by name so multiple address hosts are
- // contiguous.
- std::sort(HostFilePairs.begin(), HostFilePairs.end(), &CmpHostFilePair);
- HostDBFileContinuation::scheduleUpdate(0);
- } else if (!HostFileKeys.empty()) {
- HostDBFileContinuation::scheduleRemove(-1, 0);
- } else {
- // Nothing in new data, nothing in old data, just clean up.
- HostDBFileContinuation::finish(0);
- }
+ // Swap out hostDB's map for ours
+ hostDB.hosts_file_ptr = parsed_hosts_file_ptr;
+ // Make a new map, so we can do it all again
+ parsed_hosts_file_ptr = new RefCountedHostsFileMap();
}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/02678a1f/iocore/hostdb/P_HostDBProcessor.h
----------------------------------------------------------------------
diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h
index 6de6050..cc3200f 100644
--- a/iocore/hostdb/P_HostDBProcessor.h
+++ b/iocore/hostdb/P_HostDBProcessor.h
@@ -190,6 +190,18 @@ extern RecRawStatBlock *hostdb_rsb;
#define HOSTDB_DECREMENT_THREAD_DYN_STAT(_s, _t) RecIncrRawStatSum(hostdb_rsb, _t, (int)_s, -1);
+struct CmpConstBuffferCaseInsensitive {
+ bool operator()(ts::ConstBuffer a, ts::ConstBuffer b) { return ptr_len_casecmp(a._ptr, a._size, b._ptr, b._size) < 0;}
+};
+
+// Our own typedef for the host file mapping
+typedef std::map<ts::ConstBuffer, IpAddr, CmpConstBuffferCaseInsensitive> HostsFileMap;
+// A to hold a ref-counted map
+struct RefCountedHostsFileMap : public RefCountObj {
+ HostsFileMap hosts_file_map;
+ ats_scoped_str HostFileText;
+};
+
//
// HostDBCache (Private)
//
@@ -210,6 +222,9 @@ struct HostDBCache : public MultiCache<HostDBInfo> {
return sizeof(HostDBInfo) * 2 + 512 * hostdb_srv_enabled;
}
+ // Map to contain all of the host file overrides, initialize it to empty
+ Ptr<RefCountedHostsFileMap> hosts_file_ptr;
+
Queue<HostDBContinuation, Continuation::Link_link> pending_dns[MULTI_CACHE_PARTITIONS];
Queue<HostDBContinuation, Continuation::Link_link> &pending_dns_for_hash(INK_MD5 &md5);
HostDBCache();