You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by ja...@apache.org on 2015/07/07 02:07:46 UTC
[2/2] trafficserver git commit: Maintain and use a mapping of
hostname -> IP to implement host file support
Maintain and use a mapping of hostname -> IP to implement host file support
This allows the background thread to populate a map of overrides, which we simply inject in the lookup method-- in place of calling to DNS.
Fixes: TS-3725
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/02678a1f
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/02678a1f
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/02678a1f
Branch: refs/heads/master
Commit: 02678a1f8c9abc92985fbf6d7bb8a4b02467ba6b
Parents: 4282d33
Author: Thomas Jackson <ja...@apache.org>
Authored: Wed Jul 1 16:52:55 2015 -0700
Committer: Thomas Jackson <ja...@apache.org>
Committed: Mon Jul 6 17:07:22 2015 -0700
----------------------------------------------------------------------
iocore/hostdb/HostDB.cc | 263 +++++----------------------------
iocore/hostdb/P_HostDBProcessor.h | 15 ++
2 files changed, 51 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/02678a1f/iocore/hostdb/HostDB.cc
----------------------------------------------------------------------
diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc
index c9c54d9..838d34f 100644
--- a/iocore/hostdb/HostDB.cc
+++ b/iocore/hostdb/HostDB.cc
@@ -238,6 +238,7 @@ HostDBCache::HostDBCache()
max_hits = (1 << HOST_DB_HITS_BITS) - 1;
version.ink_major = HOST_DB_CACHE_MAJOR_VERSION;
version.ink_minor = HOST_DB_CACHE_MINOR_VERSION;
+ hosts_file_ptr = new RefCountedHostsFileMap();
}
@@ -659,6 +660,21 @@ db_mark_for(IpAddr const &ip)
HostDBInfo *
probe(ProxyMutex *mutex, HostDBMD5 const &md5, bool ignore_timeout)
{
+ // If we have an entry in our hosts file, we don't need to bother with DNS
+ Ptr<RefCountedHostsFileMap> current_host_file_map = hostDB.hosts_file_ptr;
+
+ ts::ConstBuffer hname(md5.host_name, md5.host_len);
+ HostsFileMap::iterator find_result = current_host_file_map->hosts_file_map.find(hname);
+ if (find_result != current_host_file_map->hosts_file_map.end()) {
+ // TODO: Something to make this not local :/
+ static HostDBInfo r;
+ r.round_robin = false;
+ r.reverse_dns = false;
+ r.is_srv = false;
+ ats_ip_set(r.ip(), (*find_result).second);
+ return &r;
+ }
+
ink_assert(this_ethread() == hostDB.lock_for_bucket((int)(fold_md5(md5.hash) % hostDB.buckets))->thread_holding);
if (hostdb_enable) {
uint64_t folded_md5 = fold_md5(md5.hash);
@@ -2527,19 +2543,6 @@ struct HostDBFileContinuation : public Continuation {
HostDBFileContinuation() : Continuation(0) {}
- int insertEvent(int event, void *data);
- int removeEvent(int event, void *data);
-
- /// Set the current entry in the HostDB.
- HostDBInfo *setDBEntry();
- /// Create and schedule an update continuation.
- static void scheduleUpdate(int idx, ///< Pair index to process.
- Keys *keys = 0 ///< Key table if any.
- );
- /// Create and schedule a remove continuation.
- static void scheduleRemove(int idx, ///< Index of current new key to check against.
- Keys *keys ///< new valid keys
- );
/// Finish update
static void finish(Keys *keys ///< Valid keys from update.
);
@@ -2562,21 +2565,9 @@ HostDBFileContinuation::destroy()
// proceeding at a time in any case so we might as well make these
// globals.
int HostDBFileUpdateActive = 0;
-// Contents of the host file. We keep this around because other data
-// points in to it (to minimize allocations).
-ats_scoped_str HostFileText;
-// Accumulated pairs of <IP address, host>
-std::vector<HostFilePair> HostFilePairs;
-// Entries from last update.
-HostDBFileContinuation::Keys HostFileKeys;
-/// Ordering operator for HostFilePair.
-/// We want to group first by name and then by address family.
-bool
-CmpHostFilePair(HostFilePair const &lhs, HostFilePair const &rhs)
-{
- int zret = strcasecmp(lhs.name, rhs.name);
- return zret < 0 || (0 == zret && lhs.ip < rhs.ip);
-}
+// map of hostname -> IpAddr
+Ptr<RefCountedHostsFileMap> parsed_hosts_file_ptr(new RefCountedHostsFileMap());
+
// Actual ordering doesn't matter as long as it's consistent.
bool
CmpMD5(INK_MD5 const &lhs, INK_MD5 const &rhs)
@@ -2584,185 +2575,6 @@ CmpMD5(INK_MD5 const &lhs, INK_MD5 const &rhs)
return lhs[0] < rhs[0] || (lhs[0] == rhs[0] && lhs[1] < rhs[1]);
}
-/// Finish current update.
-void
-HostDBFileContinuation::finish(Keys *keys)
-{
- HostFilePairs.clear();
- HostFileKeys.clear();
- if (keys) {
- std::swap(*keys, HostFileKeys); // put the new keys in place and dump the previous.
- delete keys;
- }
-
- HostFileText = 0;
- HostDBFileUpdateActive = 0;
-}
-
-HostDBInfo *
-HostDBFileContinuation::setDBEntry()
-{
- HostDBInfo *r;
- uint64_t folded_md5 = fold_md5(md5);
-
- // remove the old one to prevent buildup
- if (0 != (r = hostDB.lookup_block(folded_md5, hostDB.levels))) {
- hostDB.delete_block(r);
- Debug("hostdb", "Update host file entry %s - %" PRIx64 ".%" PRIx64, name, md5[0], md5[1]);
- } else {
- Debug("hostdb", "Add host file entry %s - %" PRIx64 ".%" PRIx64, name, md5[0], md5[1]);
- }
-
- r = hostDB.insert_block(folded_md5, NULL, 0);
- r->md5_high = md5[1];
- r->ip_timeout_interval = 0; // special value - no timeout.
- r->ip_timestamp = hostdb_current_interval;
- keys->push_back(md5);
- return r;
-}
-
-int
-HostDBFileContinuation::insertEvent(int, void *)
-{
- int n = HostFilePairs.size();
- HostFilePair const &first = HostFilePairs[idx];
- int last = idx + 1;
- HostDBInfo *r = 0;
-
- ink_assert(idx < n);
-
- // Get the set of addresses for this name.
- while (last < n && 0 == strcasecmp(first.name, HostFilePairs[last].name) && first.ip.family() == HostFilePairs[last].ip.family())
- ++last;
- // Address set is now (idx, last].
-
- int k = last - idx; // # of addrs for this name
- ink_assert(k > 0);
-
- r = this->setDBEntry();
- r->reverse_dns = false;
- r->is_srv = false;
- r->data.ip.assign(HostFilePairs[idx].ip);
- if (k > 1) {
- // multiple entries, need round robin
- int s = HostDBRoundRobin::size(k, false);
- HostDBRoundRobin *rr_data = static_cast<HostDBRoundRobin *>(hostDB.alloc(&r->app.rr.offset, s));
- if (rr_data) {
- int dst = 0; // index of destination RR item.
- for (int src = idx; src < last; ++src, ++dst) {
- HostDBInfo &item = rr_data->info[dst];
- item.data.ip.assign(HostFilePairs[src].ip);
- item.full = 1;
- item.round_robin = false;
- item.reverse_dns = false;
- item.is_srv = false;
- item.md5_high = r->md5_high;
- item.md5_low = r->md5_low;
- item.md5_low_low = r->md5_low_low;
- }
- r->round_robin = true;
- rr_data->good = k;
- rr_data->current = 0;
- rr_data->rrcount = dst;
- }
- } else {
- r->round_robin = false;
- }
-
- if (last < n) { // more entries to process
- // We create a new continuation rather than using this one to
- // avoid mutex issues - it is critical that the continuation use
- // the bucket mutex to guarantee a data lock for update. It's
- // unclear if changing the mutex to this continuation will do that
- // properly. Because we use a class allocator we should achieve
- // efficient allocations quickly even for large host files.
- this->scheduleUpdate(last, keys);
- } else {
- std::sort(keys->begin(), keys->end(), &CmpMD5);
- // keys->qsort(&CmpMD5);
- // Switch to removing dead entries.
- this->scheduleRemove(keys->size() - 1, keys);
- }
- // This continuation is done.
- this->destroy();
- return EVENT_DONE;
-}
-
-int
-HostDBFileContinuation::removeEvent(int, void *)
-{
- HostDBInfo *r;
- uint64_t folded_md5 = fold_md5(md5);
- Debug("hostdb", "Remove host file entry %" PRIx64 ".%" PRIx64, md5[0], md5[1]);
- if (0 != (r = hostDB.lookup_block(folded_md5, 3)))
- hostDB.delete_block(r);
- this->scheduleRemove(idx, keys);
- this->destroy();
- return EVENT_DONE;
-}
-
-void
-HostDBFileContinuation::scheduleUpdate(int idx, Keys *keys)
-{
- HostDBFileContinuation *c = hostDBFileContAllocator.alloc();
- HostFilePair &pair = HostFilePairs[idx];
- HostDBMD5 md5;
-
- md5.set_host(pair.name, strlen(pair.name));
- md5.db_mark = db_mark_for(pair.ip);
- md5.refresh();
-
- SET_CONTINUATION_HANDLER(c, &HostDBFileContinuation::insertEvent);
- c->idx = idx;
- c->name = pair.name;
- c->keys = keys ? keys : new Keys;
- c->md5 = md5.hash;
- c->mutex = hostDB.lock_for_bucket(fold_md5(md5.hash) % hostDB.buckets);
- eventProcessor.schedule_imm(c, ET_CALL, EVENT_IMMEDIATE, 0);
-}
-
-void
-HostDBFileContinuation::scheduleRemove(int idx, Keys *keys)
-{
- bool targetp = false; // Have a valid target?
- INK_MD5 md5;
-
- // See if we have a target.
- if (HostFileKeys.size() > 0) {
- md5 = HostFileKeys.back();
- HostFileKeys.pop_back();
- targetp = true;
- // Move backwards through the valid items and the previous
- // keys, removing if we have a prev key that's bigger than
- // the current valid key (which means it's not currently valid).
- while (idx >= 0) {
- if (CmpMD5((*keys)[idx], md5)) { // prev is not valid, remove
- break;
- } else if (CmpMD5(md5, (*keys)[idx])) {
- --idx; // prev is smaller, skip current and keep checking
- } else if (!HostFileKeys.empty()) {
- md5 = HostFileKeys.back();
- HostFileKeys.pop_back(); // match, move to next potential target
- --idx;
- } else { // ran out of things to remove.
- targetp = false;
- break;
- }
- }
- }
-
- if (targetp) {
- HostDBFileContinuation *c = hostDBFileContAllocator.alloc();
- SET_CONTINUATION_HANDLER(c, &HostDBFileContinuation::removeEvent);
- c->md5 = md5;
- c->idx = idx;
- c->keys = keys;
- c->mutex = hostDB.lock_for_bucket(fold_md5(c->md5) % hostDB.buckets);
- eventProcessor.schedule_imm(c, ET_CALL, EVENT_IMMEDIATE, 0);
- } else {
- self::finish(keys);
- }
-}
void
ParseHostLine(char *l)
@@ -2771,11 +2583,14 @@ ParseHostLine(char *l)
int n_elts = elts.Initialize(l, SHARE_TOKS);
// Elements should be the address then a list of host names.
// Don't use RecHttpLoadIp because the address *must* be literal.
- HostFilePair item;
- if (n_elts > 1 && 0 == item.ip.load(elts[0]) && !item.ip.isLoopback()) {
+ IpAddr ip;
+ if (n_elts > 1 && 0 == ip.load(elts[0]) && !ip.isLoopback()) {
for (int i = 1; i < n_elts; ++i) {
- item.name = elts[i];
- HostFilePairs.push_back(item);
+ ts::ConstBuffer name(elts[i], strlen(elts[i]));
+ // If we don't have an entry already (host files only support single IPs for a given name)
+ if (parsed_hosts_file_ptr->hosts_file_map.find(name) == parsed_hosts_file_ptr->hosts_file_map.end()) {
+ parsed_hosts_file_ptr->hosts_file_map[name] = ip;
+ }
}
}
}
@@ -2798,13 +2613,14 @@ ParseHostFile(char const *path)
if (0 == fstat(fd, &info)) {
// +1 in case no terminating newline
int64_t size = info.st_size + 1;
- HostFileText = static_cast<char *>(ats_malloc(size));
- if (HostFileText) {
- char *base = HostFileText;
+
+ parsed_hosts_file_ptr->HostFileText = static_cast<char *>(ats_malloc(size));
+ if (parsed_hosts_file_ptr->HostFileText) {
+ char *base = parsed_hosts_file_ptr->HostFileText;
char *limit;
- size = read(fd, HostFileText, info.st_size);
- limit = HostFileText + size;
+ size = read(fd, parsed_hosts_file_ptr->HostFileText, info.st_size);
+ limit = parsed_hosts_file_ptr->HostFileText + size;
*limit = 0;
// We need to get a list of all name/addr pairs so that we can
@@ -2833,15 +2649,8 @@ ParseHostFile(char const *path)
}
}
- if (!HostFilePairs.empty()) {
- // Need to sort by name so multiple address hosts are
- // contiguous.
- std::sort(HostFilePairs.begin(), HostFilePairs.end(), &CmpHostFilePair);
- HostDBFileContinuation::scheduleUpdate(0);
- } else if (!HostFileKeys.empty()) {
- HostDBFileContinuation::scheduleRemove(-1, 0);
- } else {
- // Nothing in new data, nothing in old data, just clean up.
- HostDBFileContinuation::finish(0);
- }
+ // Swap out hostDB's map for ours
+ hostDB.hosts_file_ptr = parsed_hosts_file_ptr;
+ // Make a new map, so we can do it all again
+ parsed_hosts_file_ptr = new RefCountedHostsFileMap();
}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/02678a1f/iocore/hostdb/P_HostDBProcessor.h
----------------------------------------------------------------------
diff --git a/iocore/hostdb/P_HostDBProcessor.h b/iocore/hostdb/P_HostDBProcessor.h
index 6de6050..cc3200f 100644
--- a/iocore/hostdb/P_HostDBProcessor.h
+++ b/iocore/hostdb/P_HostDBProcessor.h
@@ -190,6 +190,18 @@ extern RecRawStatBlock *hostdb_rsb;
#define HOSTDB_DECREMENT_THREAD_DYN_STAT(_s, _t) RecIncrRawStatSum(hostdb_rsb, _t, (int)_s, -1);
+struct CmpConstBuffferCaseInsensitive {
+ bool operator()(ts::ConstBuffer a, ts::ConstBuffer b) { return ptr_len_casecmp(a._ptr, a._size, b._ptr, b._size) < 0;}
+};
+
+// Our own typedef for the host file mapping
+typedef std::map<ts::ConstBuffer, IpAddr, CmpConstBuffferCaseInsensitive> HostsFileMap;
+// A to hold a ref-counted map
+struct RefCountedHostsFileMap : public RefCountObj {
+ HostsFileMap hosts_file_map;
+ ats_scoped_str HostFileText;
+};
+
//
// HostDBCache (Private)
//
@@ -210,6 +222,9 @@ struct HostDBCache : public MultiCache<HostDBInfo> {
return sizeof(HostDBInfo) * 2 + 512 * hostdb_srv_enabled;
}
+ // Map to contain all of the host file overrides, initialize it to empty
+ Ptr<RefCountedHostsFileMap> hosts_file_ptr;
+
Queue<HostDBContinuation, Continuation::Link_link> pending_dns[MULTI_CACHE_PARTITIONS];
Queue<HostDBContinuation, Continuation::Link_link> &pending_dns_for_hash(INK_MD5 &md5);
HostDBCache();