You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by jp...@apache.org on 2010/09/17 16:29:01 UTC
svn commit: r998151 - in /trafficserver/traffic/trunk: iocore/cache/Cache.cc
iocore/cache/CacheWrite.cc iocore/cache/P_CacheInternal.h
iocore/cache/P_CachePart.h iocore/eventsystem/I_IOBuffer.h
proxy/config/records.config.in
Author: jplevyak
Date: Fri Sep 17 14:29:00 2010
New Revision: 998151
URL: http://svn.apache.org/viewvc?rev=998151&view=rev
Log:
TS-445: make the target fragment size configurable:
# The target size of a contiguous fragment on disk.
# Acceptable values are powers of 2, e.g. 65536, 131072, 262144, 524288, 1048576, 2097152.
# Larger could waste memory on slow connections, smaller could waste seeks.
CONFIG proxy.config.cache.target_fragment_size INT 1048576
Also add fast iobuffer sizes up to 2MB and fix a minor lock hold when the
aggregation buffer is blown out (performance on error path issue).
This dramatically improves performance for large files.
Modified:
trafficserver/traffic/trunk/iocore/cache/Cache.cc
trafficserver/traffic/trunk/iocore/cache/CacheWrite.cc
trafficserver/traffic/trunk/iocore/cache/P_CacheInternal.h
trafficserver/traffic/trunk/iocore/cache/P_CachePart.h
trafficserver/traffic/trunk/iocore/eventsystem/I_IOBuffer.h
trafficserver/traffic/trunk/proxy/config/records.config.in
Modified: trafficserver/traffic/trunk/iocore/cache/Cache.cc
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/cache/Cache.cc?rev=998151&r1=998150&r2=998151&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/cache/Cache.cc (original)
+++ trafficserver/traffic/trunk/iocore/cache/Cache.cc Fri Sep 17 14:29:00 2010
@@ -65,15 +65,15 @@ int cache_config_vary_on_user_agent = 0;
int cache_config_select_alternate = 1;
int cache_config_max_doc_size = 0;
int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE;
-int64 cache_config_ram_cache_cutoff = 1048576; // 1 MB
-//int64 cache_config_ram_cache_mixt_cutoff = 1048576; // 1 MB
+int64 cache_config_ram_cache_cutoff = AGG_SIZE;
int cache_config_max_disk_errors = 5;
-int cache_config_agg_write_backlog = 5242880;
#ifdef HIT_EVACUATE
int cache_config_hit_evacuate_percent = 10;
int cache_config_hit_evacuate_size_limit = 0;
#endif
int cache_config_force_sector_size = 0;
+int cache_config_target_fragment_size = 1048576;
+int cache_config_agg_write_backlog = AGG_SIZE * 2;
int cache_config_enable_checksum = 0;
int cache_config_alt_rewrite_max_size = 4096;
int cache_config_read_while_writer = 0;
@@ -2630,14 +2630,6 @@ ink_cache_init(ModuleVersion v)
Debug("cache_init", "cache_config_ram_cache_cutoff = %lld = %lldMb",
cache_config_ram_cache_cutoff, cache_config_ram_cache_cutoff / (1024 * 1024));
-#if 0
- IOCORE_RegisterConfigInteger(RECT_CONFIG,
- "proxy.config.cache.ram_cache_mixt_cutoff", 1048576, RECU_DYNAMIC, RECC_NULL, NULL);
- IOCORE_EstablishStaticConfigInteger(cache_config_ram_cache_mixt_cutoff, "proxy.config.cache.ram_cache_mixt_cutoff");
- Debug("cache_init", "proxy.config.cache.ram_cache_mixt_cutoff = %lld = %lldMb",
- cache_config_ram_cache_mixt_cutoff, cache_config_ram_cache_mixt_cutoff / (1024 * 1024));
-#endif
-
IOCORE_RegisterConfigInteger(RECT_CONFIG, "proxy.config.cache.permit.pinning", 0, RECU_DYNAMIC, RECC_NULL, NULL);
IOCORE_EstablishStaticConfigInt32(cache_config_permit_pinning, "proxy.config.cache.permit.pinning");
Debug("cache_init", "proxy.config.cache.permit.pinning = %d", cache_config_permit_pinning);
@@ -2691,6 +2683,8 @@ ink_cache_init(ModuleVersion v)
#endif
IOCORE_RegisterConfigInteger(RECT_CONFIG, "proxy.config.cache.force_sector_size", 0, RECU_DYNAMIC, RECC_NULL, NULL);
IOCORE_EstablishStaticConfigInt32(cache_config_force_sector_size, "proxy.config.cache.force_sector_size");
+ IOCORE_RegisterConfigInteger(RECT_CONFIG, "proxy.config.cache.target_fragment_size", DEFAULT_TARGET_FRAGMENT_SIZE, RECU_DYNAMIC, RECC_NULL, NULL);
+ IOCORE_EstablishStaticConfigInt32(cache_config_target_fragment_size, "proxy.config.cache.target_fragment_size");
#ifdef HTTP_CACHE
extern int url_hash_method;
IOCORE_RegisterConfigInteger(RECT_CONFIG, "proxy.config.cache.url_hash_method", 1, RECU_RESTART_TS, RECC_NULL, NULL);
Modified: trafficserver/traffic/trunk/iocore/cache/CacheWrite.cc
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/cache/CacheWrite.cc?rev=998151&r1=998150&r2=998151&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/cache/CacheWrite.cc (original)
+++ trafficserver/traffic/trunk/iocore/cache/CacheWrite.cc Fri Sep 17 14:29:00 2010
@@ -1339,6 +1339,10 @@ CacheVC::openWriteWriteDone(int event, E
return openWriteMain(event, e);
}
+static inline int target_fragment_size() {
+ return cache_config_target_fragment_size - sizeofDoc;
+}
+
int
CacheVC::openWriteMain(int event, Event *e)
{
@@ -1384,11 +1388,12 @@ Lagain:
total_len += avail;
}
length = (uint64)towrite;
- if (length > TARGET_FRAG_SIZE && length < SHRINK_TARGET_FRAG_SIZE)
- write_len = TARGET_FRAG_SIZE;
+ if (length > target_fragment_size() &&
+ (length < target_fragment_size() + target_fragment_size() / 4))
+ write_len = target_fragment_size();
else
write_len = length;
- bool not_writing = towrite != ntodo && towrite < TARGET_FRAG_SIZE;
+ bool not_writing = towrite != ntodo && towrite < target_fragment_size();
if (!called_user) {
if (not_writing) {
called_user = 1;
Modified: trafficserver/traffic/trunk/iocore/cache/P_CacheInternal.h
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/cache/P_CacheInternal.h?rev=998151&r1=998150&r2=998151&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/cache/P_CacheInternal.h (original)
+++ trafficserver/traffic/trunk/iocore/cache/P_CacheInternal.h Fri Sep 17 14:29:00 2010
@@ -227,6 +227,7 @@ extern int cache_config_hit_evacuate_per
extern int cache_config_hit_evacuate_size_limit;
#endif
extern int cache_config_force_sector_size;
+extern int cache_config_target_fragment_size;
// CacheVC
struct CacheVC: public CacheVConnection
@@ -690,7 +691,7 @@ CacheVC::handleWriteLock(int event, Even
trigger = mutex->thread_holding->schedule_in_local(this, MUTEX_RETRY_DELAY);
return EVENT_CONT;
}
- ret = handleWrite(event, e);
+ ret = handleWrite(EVENT_CALL, e);
}
if (ret == EVENT_RETURN)
return handleEvent(AIO_EVENT_DONE, 0);
Modified: trafficserver/traffic/trunk/iocore/cache/P_CachePart.h
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/cache/P_CachePart.h?rev=998151&r1=998150&r2=998151&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/cache/P_CachePart.h (original)
+++ trafficserver/traffic/trunk/iocore/cache/P_CachePart.h Fri Sep 17 14:29:00 2010
@@ -43,9 +43,7 @@
#define MAX_PART_SIZE ((off_t)512 * 1024 * 1024 * 1024 * 1024)
#define STORE_BLOCKS_PER_CACHE_BLOCK (STORE_BLOCK_SIZE / CACHE_BLOCK_SIZE)
#define MAX_PART_BLOCKS (MAX_PART_SIZE / CACHE_BLOCK_SIZE)
-#define TARGET_FRAG_SIZE (DEFAULT_MAX_BUFFER_SIZE - sizeofDoc)
-#define SHRINK_TARGET_FRAG_SIZE (DEFAULT_MAX_BUFFER_SIZE + (DEFAULT_MAX_BUFFER_SIZE/4))
-#define MAX_FRAG_SIZE ((256 * 1024) - sizeofDoc)
+#define MAX_FRAG_SIZE (AGG_SIZE - sizeofDoc) // true max
#define LEAVE_FREE DEFAULT_MAX_BUFFER_SIZE
#define PIN_SCAN_EVERY 16 // scan every 1/16 of disk
#define PART_HASH_TABLE_SIZE 32707
@@ -56,6 +54,7 @@
#define AIO_NOT_IN_PROGRESS 0
#define AIO_AGG_WRITE_IN_PROGRESS -1
#define AUTO_SIZE_RAM_CACHE -1 // 1-1 with directory size
+#define DEFAULT_TARGET_FRAGMENT_SIZE (65536 - sizeofDoc) // 64k
#define dir_offset_evac_bucket(_o) \
Modified: trafficserver/traffic/trunk/iocore/eventsystem/I_IOBuffer.h
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/iocore/eventsystem/I_IOBuffer.h?rev=998151&r1=998150&r2=998151&view=diff
==============================================================================
--- trafficserver/traffic/trunk/iocore/eventsystem/I_IOBuffer.h (original)
+++ trafficserver/traffic/trunk/iocore/eventsystem/I_IOBuffer.h Fri Sep 17 14:29:00 2010
@@ -93,7 +93,12 @@ enum AllocType
#define BUFFER_SIZE_INDEX_16K 7
#define BUFFER_SIZE_INDEX_32K 8
#define BUFFER_SIZE_INDEX_64K 9
-#define MAX_BUFFER_SIZE_INDEX 9
+#define BUFFER_SIZE_INDEX_128K 10
+#define BUFFER_SIZE_INDEX_256K 11
+#define BUFFER_SIZE_INDEX_512K 12
+#define BUFFER_SIZE_INDEX_1M 13
+#define BUFFER_SIZE_INDEX_2M 14
+#define MAX_BUFFER_SIZE_INDEX 14
#define DEFAULT_BUFFER_SIZES (MAX_BUFFER_SIZE_INDEX+1)
#define BUFFER_SIZE_FOR_INDEX(_i) (DEFAULT_BUFFER_BASE_SIZE * (1 << (_i)))
Modified: trafficserver/traffic/trunk/proxy/config/records.config.in
URL: http://svn.apache.org/viewvc/trafficserver/traffic/trunk/proxy/config/records.config.in?rev=998151&r1=998150&r2=998151&view=diff
==============================================================================
--- trafficserver/traffic/trunk/proxy/config/records.config.in (original)
+++ trafficserver/traffic/trunk/proxy/config/records.config.in Fri Sep 17 14:29:00 2010
@@ -315,7 +315,7 @@ CONFIG proxy.config.cache.permit.pinning
# (approximately 1 MB of RAM cache per GB of disk cache)
# alternatively, set to a fixed value such as 20971520 (20MB)
CONFIG proxy.config.cache.ram_cache.size INT -1
-CONFIG proxy.config.cache.ram_cache_cutoff INT 1048576
+CONFIG proxy.config.cache.ram_cache_cutoff INT 4194304
# Replacement algorithm
# 0 : Clocked Least Frequently Used by Size (CLFUS) w/optional compression
# 1 : LRU w/o optional compression - trivially simple
@@ -332,6 +332,10 @@ CONFIG proxy.config.cache.ram_cache.comp
# The default value for 'proxy.config.cache.vary_on_user_agent' is 0.
# (0 disables the maximum number of alts check)
CONFIG proxy.config.cache.limits.http.max_alts INT 5
+ # The target size of a contiguous fragment on disk.
+ # Acceptable values are powers of 2, e.g. 65536, 131072, 262144, 524288, 1048576, 2097152.
+ # Larger could waste memory on slow connections, smaller could waste seeks.
+CONFIG proxy.config.cache.target_fragment_size INT 1048576
# The maximum size of a document that will be stored in the cache.
# (0 disables the maximum document size check)
CONFIG proxy.config.cache.max_doc_size INT 0