You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2011/05/27 04:36:07 UTC
svn commit: r1128140 - in /trafficserver/traffic/trunk: iocore/hostdb/ mgmt/
mgmt/cli/ proxy/config/
Author: zwoop
Date: Fri May 27 02:36:07 2011
New Revision: 1128140
URL: http://svn.apache.org/viewvc?rev=1128140&view=rev
Log:
TS-805 HostDB wastes a lot of storage for round-robin entries, and the calculations of size are off
Modified:
trafficserver/traffic/trunk/iocore/hostdb/HostDB.cc
trafficserver/traffic/trunk/iocore/hostdb/I_HostDBProcessor.h
trafficserver/traffic/trunk/iocore/hostdb/MultiCache.cc
trafficserver/traffic/trunk/iocore/hostdb/P_HostDBProcessor.h
trafficserver/traffic/trunk/iocore/hostdb/P_MultiCache.h
trafficserver/traffic/trunk/mgmt/RecordsConfig.cc
trafficserver/traffic/trunk/mgmt/cli/Makefile.am
trafficserver/traffic/trunk/proxy/config/records.config.default.in
Modified: trafficserver/traffic/trunk/iocore/hostdb/HostDB.cc
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/hostdb/HostDB.cc?rev=1128140&r1=1128139&r2=1128140&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/hostdb/HostDB.cc (original)
+++ trafficserver/traffic/trunk/iocore/hostdb/HostDB.cc Fri May 27 02:36:07 2011
@@ -1989,23 +1989,21 @@ bool HostDBInfo::match(INK_MD5 & md5, in
if (md5[1] != md5_high)
return false;
- uint64_t
- folded_md5 = fold_md5(md5);
- uint64_t
- ttag = folded_md5 / buckets;
+ uint64_t folded_md5 = fold_md5(md5);
+ uint64_t ttag = folded_md5 / buckets;
if (!ttag)
ttag = 1;
+
struct
{
- unsigned int
- md5_low_low:
- 24;
- unsigned int
- md5_low;
+ unsigned int md5_low_low:24;
+ unsigned int md5_low;
} tmp;
+
tmp.md5_low_low = (unsigned int) ttag;
tmp.md5_low = (unsigned int) (ttag >> 24);
+
return tmp.md5_low_low == md5_low_low && tmp.md5_low == md5_low;
}
Modified: trafficserver/traffic/trunk/iocore/hostdb/I_HostDBProcessor.h
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/hostdb/I_HostDBProcessor.h?rev=1128140&r1=1128139&r2=1128140&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/hostdb/I_HostDBProcessor.h (original)
+++ trafficserver/traffic/trunk/iocore/hostdb/I_HostDBProcessor.h Fri May 27 02:36:07 2011
@@ -120,8 +120,7 @@ struct HostDBRoundRobin;
struct HostDBInfo
{
// Public Interface
- unsigned int &ip()
- {
+ unsigned int &ip() {
return data.ip;
}
@@ -269,27 +268,13 @@ struct HostDBInfo
sockaddr_in6 ip6;
- bool failed() {
- return !ip();
- }
+ bool failed() { return !ip(); }
+ void set_failed() { ip() = 0; }
- void set_failed()
- {
- ip() = 0;
- }
+ void set_deleted() { deleted = 1; }
+ bool is_deleted() const { return deleted; }
- void set_deleted()
- {
- deleted = 1;
- }
-
- bool is_deleted() {
- return deleted;
- }
-
- bool is_empty() {
- return !full;
- }
+ bool is_empty() const { return !full; }
void set_empty()
{
@@ -307,6 +292,7 @@ struct HostDBInfo
void set_full(uint64_t folded_md5, int buckets)
{
uint64_t ttag = folded_md5 / buckets;
+
if (!ttag)
ttag = 1;
md5_low_low = (unsigned int) ttag;
@@ -335,8 +321,8 @@ struct HostDBInfo
int heap_size();
int *heap_offset_ptr();
-HostDBInfo():
- srv_weight(0), srv_priority(0), srv_port(0), srv_count(0), is_srv(0),
+HostDBInfo()
+ : srv_weight(0), srv_priority(0), srv_port(0), srv_count(0), is_srv(0),
ip_timestamp(0),
ip_timeout_interval(0), full(0), backed(0), deleted(0), hits(0), round_robin(0), reverse_dns(0), md5_low_low(0),
md5_low(0), md5_high(0) {
@@ -378,9 +364,10 @@ struct HostDBRoundRobin
return (int) ((sizeof(HostDBRoundRobin)) -
(sizeof(HostDBInfo) * (HOST_DB_MAX_ROUND_ROBIN_INFO - nn)) -
(sizeof(char) * MAXDNAME * (HOST_DB_MAX_ROUND_ROBIN_INFO - nn)));
- } else
- {
- return (int) (sizeof(HostDBRoundRobin) - sizeof(HostDBInfo) * (HOST_DB_MAX_ROUND_ROBIN_INFO - nn));
+ } else {
+ return (int) (sizeof(HostDBRoundRobin) -
+ sizeof(rr_srv_hosts) -
+ sizeof(HostDBInfo) * (HOST_DB_MAX_ROUND_ROBIN_INFO - nn));
}
}
Modified: trafficserver/traffic/trunk/iocore/hostdb/MultiCache.cc
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/hostdb/MultiCache.cc?rev=1128140&r1=1128139&r2=1128140&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/hostdb/MultiCache.cc (original)
+++ trafficserver/traffic/trunk/iocore/hostdb/MultiCache.cc Fri May 27 02:36:07 2011
@@ -188,13 +188,14 @@ MultiCacheBase::initialize(Store * astor
unsigned int blocks = (size + (STORE_BLOCK_SIZE - 1)) / STORE_BLOCK_SIZE;
- heap_size = int ((float) totalelements * estimated_heap_bytes_per_entry() * 4.0);
+ heap_size = int ((float)totalelements * estimated_heap_bytes_per_entry());
blocks += bytes_to_blocks(heap_size);
blocks += 1; // header
-
totalsize = (int64_t) blocks *(int64_t) STORE_BLOCK_SIZE;
+ Debug("multicache", "heap_size = %d, totalelements = %d, totalsize = %d", heap_size, totalelements, totalsize);
+
//
// Spread alloc from the store (using storage that can be mmapped)
//
@@ -1335,9 +1336,9 @@ void *
MultiCacheBase::alloc(int *poffset, int asize)
{
int h = heap_halfspace;
- int size = (asize + MULTI_CACHE_HEAP_ALIGNMENT - 1)
- & ~(MULTI_CACHE_HEAP_ALIGNMENT - 1);
+ int size = (asize + MULTI_CACHE_HEAP_ALIGNMENT - 1) & ~(MULTI_CACHE_HEAP_ALIGNMENT - 1);
int o = ink_atomic_increment((int *) &heap_used[h], size);
+
if (o + size > halfspace_size()) {
ink_atomic_increment((int *) &heap_used[h], -size);
ink_assert(!"out of space");
Modified: trafficserver/traffic/trunk/iocore/hostdb/P_HostDBProcessor.h
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/hostdb/P_HostDBProcessor.h?rev=1128140&r1=1128139&r2=1128140&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/hostdb/P_HostDBProcessor.h (original)
+++ trafficserver/traffic/trunk/iocore/hostdb/P_HostDBProcessor.h Fri May 27 02:36:07 2011
@@ -137,11 +137,10 @@ struct HostDBCache: public MultiCache<Ho
{
return NEW(new HostDBCache);
}
- virtual float estimated_heap_bytes_per_entry()
- {
- //return 16.0;
- return 24.0;
- }
+
+ // This accounts for an average of 2 HostDBInfo per DNS cache (for round-robin etc.)
+ virtual size_t estimated_heap_bytes_per_entry() const { return sizeof(HostDBInfo) * 2; }
+
Queue<HostDBContinuation, Continuation::Link_link> pending_dns[MULTI_CACHE_PARTITIONS];
Queue<HostDBContinuation, Continuation::Link_link> &pending_dns_for_hash(INK_MD5 & md5);
HostDBCache();
@@ -269,7 +268,6 @@ typedef int (HostDBContinuation::*HostDB
struct HostDBContinuation: public Continuation
{
-
Action action;
unsigned int ip;
unsigned int ttl;
Modified: trafficserver/traffic/trunk/iocore/hostdb/P_MultiCache.h
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/hostdb/P_MultiCache.h?rev=1128140&r1=1128139&r2=1128140&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/hostdb/P_MultiCache.h (original)
+++ trafficserver/traffic/trunk/iocore/hostdb/P_MultiCache.h Fri May 27 02:36:07 2011
@@ -295,10 +295,7 @@ struct MultiCacheBase: public MultiCache
return NULL;
}
- virtual float estimated_heap_bytes_per_entry()
- {
- return 0.0;
- }
+ virtual size_t estimated_heap_bytes_per_entry() const { return 0; }
void print_info(FILE * fp);
Modified: trafficserver/traffic/trunk/mgmt/RecordsConfig.cc
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/mgmt/RecordsConfig.cc?rev=1128140&r1=1128139&r2=1128140&view=diff
==============================================================================
--- trafficserver/traffic/trunk/mgmt/RecordsConfig.cc (original)
+++ trafficserver/traffic/trunk/mgmt/RecordsConfig.cc Fri May 27 02:36:07 2011
@@ -1058,7 +1058,7 @@ RecordElement RecordsConfig[] = {
{RECT_CONFIG, "proxy.config.hostdb.filename", RECD_STRING, "host.db", RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
,
// # in entries, may not be changed while running
- {RECT_CONFIG, "proxy.config.hostdb.size", RECD_INT, "200000", RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
+ {RECT_CONFIG, "proxy.config.hostdb.size", RECD_INT, "120000", RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
,
{RECT_CONFIG, "proxy.config.hostdb.storage_path", RECD_STRING, TS_BUILD_CACHEDIR, RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
,
Modified: trafficserver/traffic/trunk/mgmt/cli/Makefile.am
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/mgmt/cli/Makefile.am?rev=1128140&r1=1128139&r2=1128140&view=diff
==============================================================================
--- trafficserver/traffic/trunk/mgmt/cli/Makefile.am (original)
+++ trafficserver/traffic/trunk/mgmt/cli/Makefile.am Fri May 27 02:36:07 2011
@@ -70,7 +70,7 @@ libtrafficshell_a_SOURCES = \
UtilCmds.cc \
UtilCmds.h
-traffic_shell_LDFLAGS = $(EXTRA_CXX_LDFLAGS) @LIBTOOL_LINK_FLAGS@
+traffic_shell_LDFLAGS = @EXTRA_CXX_LDFLAGS@ @LIBTOOL_LINK_FLAGS@
traffic_shell_LDADD = \
libtrafficshell.a \
$(top_builddir)/mgmt/api/remote/libtsmgmt.la \
Modified: trafficserver/traffic/trunk/proxy/config/records.config.default.in
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/proxy/config/records.config.default.in?rev=1128140&r1=1128139&r2=1128140&view=diff
==============================================================================
--- trafficserver/traffic/trunk/proxy/config/records.config.default.in (original)
+++ trafficserver/traffic/trunk/proxy/config/records.config.default.in Fri May 27 02:36:07 2011
@@ -374,7 +374,7 @@ CONFIG proxy.config.dns.validate_query_n
# in entries, may not be changed while running
# note that in order to increase hostdb.size, hostdb.storage_size should
# also be increase. These are best guesses, you will have to monitor this.
-CONFIG proxy.config.hostdb.size INT 150000
+CONFIG proxy.config.hostdb.size INT 120000
CONFIG proxy.config.hostdb.storage_size INT 32M
# ttl modes:
# 0 = obey