You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by am...@apache.org on 2015/04/19 19:45:15 UTC
[1/8] trafficserver git commit: First pass on removing range
processing from HttpSM.
Repository: trafficserver
Updated Branches:
refs/heads/ts-974-multi-range-read [created] 2ddfc7ab5
First pass on removing range processing from HttpSM.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/ae369eea
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/ae369eea
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/ae369eea
Branch: refs/heads/ts-974-multi-range-read
Commit: ae369eeade78ed9a80c80ead6d9a6aa4775ce48d
Parents: d33b625
Author: Alan M. Carroll <so...@yahoo-inc.com>
Authored: Tue Dec 2 10:54:44 2014 -0600
Committer: Alan M. Carroll <so...@yahoo-inc.com>
Committed: Sat Dec 6 11:56:03 2014 -0600
----------------------------------------------------------------------
proxy/http/HttpSM.cc | 4 ++++
proxy/http/HttpTransact.cc | 6 +++++-
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/ae369eea/proxy/http/HttpSM.cc
----------------------------------------------------------------------
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index 5f07f8c..d1edc50 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -4351,6 +4351,9 @@ HttpSM::do_range_parse(MIMEField *range_field)
void
HttpSM::do_range_setup_if_necessary()
{
+# if 1
+ t_state.range_setup = HttpTransact::RANGE_NONE;
+# else
MIMEField *field;
INKVConnInternal *range_trans;
int field_content_type_len = -1;
@@ -4393,6 +4396,7 @@ HttpSM::do_range_setup_if_necessary()
}
}
}
+# endif
}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/ae369eea/proxy/http/HttpTransact.cc
----------------------------------------------------------------------
diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc
index fe65f81..c348c29a 100644
--- a/proxy/http/HttpTransact.cc
+++ b/proxy/http/HttpTransact.cc
@@ -2811,7 +2811,7 @@ HttpTransact::build_response_from_cache(State* s, HTTPWarningCode warning_code)
// send back the full document to the client.
DebugTxn("http_trans", "[build_response_from_cache] Match! Serving full document.");
s->cache_info.action = CACHE_DO_SERVE;
-
+# if 0
// Check if cached response supports Range. If it does, append
// Range transformation plugin
// only if the cached response is a 200 OK
@@ -2845,6 +2845,10 @@ HttpTransact::build_response_from_cache(State* s, HTTPWarningCode warning_code)
build_response(s, cached_response, &s->hdr_info.client_response, s->client_info.http_version);
}
s->next_action = SM_ACTION_SERVE_FROM_CACHE;
+# else
+ build_response(s, cached_response, &s->hdr_info.client_response, s->client_info.http_version);
+ s->next_action = SM_ACTION_SERVE_FROM_CACHE;
+# endif
}
// If the client request is a HEAD, then serve the header from cache.
else if (s->method == HTTP_WKSIDX_HEAD) {
[8/8] trafficserver git commit: Full read multi range support passes
basic testing.
Posted by am...@apache.org.
Full read multi range support passes basic testing.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/2ddfc7ab
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/2ddfc7ab
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/2ddfc7ab
Branch: refs/heads/ts-974-multi-range-read
Commit: 2ddfc7ab52b4f24593a813e123e7dacb754eac58
Parents: 67261f1
Author: Alan M. Carroll <so...@yahoo-inc.com>
Authored: Sat Dec 6 00:27:32 2014 -0600
Committer: Alan M. Carroll <so...@yahoo-inc.com>
Committed: Sat Dec 6 11:56:04 2014 -0600
----------------------------------------------------------------------
iocore/cache/CacheHttp.cc | 76 ++++++++----
iocore/cache/CacheRead.cc | 221 +++++++++++++++++++----------------
iocore/cache/I_Cache.h | 7 ++
iocore/cache/P_CacheHttp.h | 127 ++++++++++++++------
iocore/cache/P_CacheInternal.h | 5 +-
iocore/cluster/P_ClusterCache.h | 3 +
lib/ts/InkErrno.h | 3 +-
proxy/hdrs/HTTP.cc | 137 +++++++++++++++++++---
proxy/hdrs/HTTP.h | 78 +++++++++----
proxy/http/HttpSM.cc | 3 +-
proxy/http/HttpTransact.cc | 5 +-
11 files changed, 469 insertions(+), 196 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/iocore/cache/CacheHttp.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheHttp.cc b/iocore/cache/CacheHttp.cc
index 4d82c18..b5fa82d 100644
--- a/iocore/cache/CacheHttp.cc
+++ b/iocore/cache/CacheHttp.cc
@@ -259,19 +259,58 @@ CacheHTTPInfoVector::get_handles(const char *buf, int length, RefCountObj * bloc
/*-------------------------------------------------------------------------
-------------------------------------------------------------------------*/
+void
+CacheRange::clear()
+{
+ _offset = 0;
+ _idx = -1;
+ _pending_range_shift_p = false;
+ _ct_field = NULL; // need to do real cleanup at some point.
+ _r.clear();
+}
+
+bool
+CacheRange::init(HTTPHdr* req)
+{
+ bool zret = true;
+ MIMEField* rf = req->field_find(MIME_FIELD_RANGE, MIME_LEN_RANGE);
+ if (rf) {
+ int len;
+ char const* val = rf->value_get(&len);
+ zret = _r.parse(val, len);
+ }
+ return zret;
+}
+
+bool
+CacheRange::apply(uint64_t len)
+{
+ bool zret = _r.apply(len);
+ if (zret) {
+ _len = len;
+ if (_r.hasRanges()) {
+ _offset = _r[_idx = 0]._min;
+ if (_r.isMulti()) _pending_range_shift_p = true;
+ }
+ }
+ return zret;
+}
+
uint64_t
CacheRange::consume(uint64_t size)
{
- switch (_r->_state) {
+ switch (_r._state) {
case HTTPRangeSpec::EMPTY: _offset += size; break;
- case HTTPRangeSpec::SINGLE: _offset += std::min(size, (_r->_single._max - _offset) + 1 ); break;
+ case HTTPRangeSpec::SINGLE: _offset += std::min(size, (_r._single._max - _offset) + 1 ); break;
case HTTPRangeSpec::MULTI:
- while (size && _idx < static_cast<int>(_r->count())) {
- uint64_t r = std::min(size, ((*_r)[_idx]._max - _offset) + 1);
- _offset += r;
- size -= r;
- if (_offset > (*_r)[_idx]._max)
- _offset = (*_r)[++_idx]._min;
+ ink_assert(_idx < static_cast<int>(_r.count()));
+ // Must not consume more than 1 range or the boundary strings won't get sent.
+ ink_assert(!_pending_range_shift_p);
+ ink_assert(size <= (_r[_idx]._max - _offset) + 1);
+ _offset += size;
+ if (_offset > _r[_idx]._max && ++_idx < static_cast<int>(_r.count())) {
+ _offset = _r[_idx]._min;
+ _pending_range_shift_p = true;
}
break;
default: break;
@@ -280,25 +319,22 @@ CacheRange::consume(uint64_t size)
return _offset;
}
-void
+CacheRange&
CacheRange::generateBoundaryStr(CacheKey const& key)
{
- snprintf(_boundary, sizeof(_boundary), "%08" PRIu64 "%08" PRIu64 "..%08" PRIu64
- , key.slice64(0), key.slice64(1), this_ethread()->generator.random()
- );
-}
-
-bool
-CacheRange::setContentType(HTTPHdr* header)
-{
- _ct_field = header->field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
- return NULL != _ct_field;
+ uint64_t rnd = this_ethread()->generator.random();
+ snprintf(_boundary, sizeof(_boundary), "%016" PRIx64 "%016" PRIx64 "..%016" PRIx64, key.slice64(0), key.slice64(1), rnd);
+ // GAH! snprintf null terminates so we can't actually print the last nybble that way and all of
+ // the internal hex converters do the same thing. This is crazy code I need to fix at some point.
+ // It is critical to print every nybble or the content lengths won't add up.
+ _boundary[HTTP_RANGE_BOUNDARY_LEN-1] = "0123456789abcdef"[rnd & 0xf];
+ return *this;
}
uint64_t
CacheRange::calcContentLength() const
{
- return _r->calcContentLength(_len, _ct_field->m_len_value);
+ return _r.calcContentLength(_len, _ct_field ? _ct_field->m_len_value : 0);
}
/*-------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/iocore/cache/CacheRead.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc
index ac5e13e..c8ff47b 100644
--- a/iocore/cache/CacheRead.cc
+++ b/iocore/cache/CacheRead.cc
@@ -184,6 +184,18 @@ CacheVC::get_http_content_size()
return resp_range.calcContentLength();
}
+HTTPRangeSpec&
+CacheVC::get_http_range_spec()
+{
+ return resp_range.getRangeSpec();
+}
+
+bool
+CacheVC::is_http_partial_content()
+{
+ return resp_range.hasRanges();
+}
+
int
CacheVC::openReadFromWriterFailure(int event, Event * e)
{
@@ -690,17 +702,64 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
{
cancel_trigger();
Doc *doc = (Doc *) buf->data();
- int64_t ntodo = vio.ntodo();
- int64_t bytes = doc->len - doc_pos;
+ int64_t bytes = vio.ntodo();
IOBufferBlock *b = NULL;
+ uint64_t target_offset = resp_range.getOffset();
+ uint64_t lower_bound = frag_upper_bound - doc->data_len();
+
+ if (bytes <= 0)
+ return EVENT_CONT;
+
+ // Start shipping
+ while (bytes > 0 && lower_bound <= target_offset && target_offset < frag_upper_bound) {
+ if (vio.buffer.writer()->max_read_avail() > vio.buffer.writer()->water_mark && vio.ndone) // wait for reader
+ return EVENT_CONT;
+
+ if (resp_range.hasPendingRangeShift()) { // in new range, shift to start location.
+ int b_len;
+ char const* b_str = resp_range.getBoundaryStr(&b_len);
+ size_t r_idx = resp_range.getIdx();
+
+ doc_pos = doc->prefix_len() + (target_offset - lower_bound);
+
+ vio.ndone += HTTPRangeSpec::writePartBoundary(vio.buffer.writer(), b_str, b_len
+ , doc_len, resp_range[r_idx]._min, resp_range[r_idx]._max
+ , resp_range.getContentTypeField(), r_idx >= (resp_range.count() - 1)
+ );
+ resp_range.consumeRangeShift();
+ Debug("amc", "Range boundary for range %" PRIu64, r_idx);
+ }
+
+ bytes = std::min(doc->len - doc_pos, static_cast<int64_t>(resp_range.getRemnantSize()));
+ bytes = std::min(bytes, vio.ntodo());
+ if (bytes > 0) {
+ b = new_IOBufferBlock(buf, bytes, doc_pos);
+ b->_buf_end = b->_end;
+ vio.buffer.writer()->append_block(b);
+ vio.ndone += bytes;
+ doc_pos += bytes;
+ resp_range.consume(bytes);
+ Debug("amc", "shipped %" PRId64 " bytes at target offset %" PRIu64, bytes, target_offset);
+ target_offset = resp_range.getOffset();
+ }
+
+ if (vio.ntodo() <= 0)
+ return calluser(VC_EVENT_READ_COMPLETE);
+ else if (calluser(VC_EVENT_READ_READY) == EVENT_DONE)
+ return EVENT_DONE;
+ }
+
+
#ifdef HTTP_CACHE
- if (resp_range.isActive()) {
- int target = -1; // target fragment index.
- uint64_t target_offset = resp_range.getOffset();
- uint64_t lower_bound = frag_upper_bound - doc->data_len();
+ if (resp_range.getRemnantSize()) {
+ HTTPInfo::FragOffset* frags = alternate.get_frag_table();
+ int n_frags = alternate.get_frag_offset_count();
- if (target_offset < lower_bound || frag_upper_bound <= target_offset) {
- HTTPInfo::FragOffset* frags = alternate.get_frag_table();
+ // Quick check for offset in next fragment - very common
+ if (target_offset >= frag_upper_bound && (!frags || fragment >= (n_frags-1) || target_offset < frags[fragment])) {
+ Debug("amc", "Non-seeking continuation to next fragment");
+ } else {
+ int target = -1; // target fragment index.
if (is_debug_tag_set("amc")) {
char b[33], c[33];
@@ -710,100 +769,60 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
);
}
- if (frags) {
- int n_frags = alternate.get_frag_offset_count();
- target = this->frag_idx_for_offset(frags, n_frags, target_offset);
- this->update_key_to_frag_idx(target);
- /// one frag short, because it gets bumped when the fragment is actually read.
- frag_upper_bound = target > 0 ? frags[target-1] : 0;
- } else { // we don't support non-monotonic forward requests on objects with no frag table.
- ink_release_assert(target_offset >= frag_upper_bound);
- }
- goto Lread;
- } else {
- // Adjust fragment starting offset to account for target offset.
- int r_doc_pos = doc->prefix_len() + (target_offset - lower_bound);
- if (doc_pos < r_doc_pos) {
- doc_pos = r_doc_pos;
- bytes = doc->len - doc_pos;
- }
- bytes = std::min(bytes, static_cast<int64_t>(resp_range.getRemnantSize()));
+ target = this->frag_idx_for_offset(frags, n_frags, target_offset);
+ this->update_key_to_frag_idx(target);
+ /// one frag short, because it gets bumped when the fragment is actually read.
+ frag_upper_bound = target > 0 ? frags[target-1] : 0;
+ Debug("amc", "Fragment seek from %d to %d target offset %" PRIu64, fragment - 1, target, target_offset);
}
+ }
#endif
+
+ if (vio.ntodo() > 0 && 0 == resp_range.getRemnantSize())
+ // reached the end of the document and the user still wants more
+ return calluser(VC_EVENT_EOS);
+ last_collision = 0;
+ writer_lock_retry = 0;
+ // if the state machine calls reenable on the callback from the cache,
+ // we set up a schedule_imm event. The openReadReadDone discards
+ // EVENT_IMMEDIATE events. So, we have to cancel that trigger and set
+ // a new EVENT_INTERVAL event.
+ cancel_trigger();
+ CACHE_TRY_LOCK(lock, vol->mutex, mutex->thread_holding);
+ if (!lock.is_locked()) {
+ SET_HANDLER(&CacheVC::openReadMain);
+ VC_SCHED_LOCK_RETRY();
}
- if (ntodo <= 0)
+ if (dir_probe(&key, vol, &dir, &last_collision)) {
+ SET_HANDLER(&CacheVC::openReadReadDone);
+ int ret = do_read_call(&key);
+ if (ret == EVENT_RETURN)
+ goto Lcallreturn;
return EVENT_CONT;
- if (vio.buffer.writer()->max_read_avail() > vio.buffer.writer()->water_mark && vio.ndone) // initiate read of first block
- return EVENT_CONT;
- if ((bytes <= 0) && vio.ntodo() >= 0)
- goto Lread;
- // Do the write to downstream consumers.
- if (bytes > vio.ntodo())
- bytes = vio.ntodo();
- b = new_IOBufferBlock(buf, bytes, doc_pos);
- b->_buf_end = b->_end;
- vio.buffer.writer()->append_block(b);
- vio.ndone += bytes;
- doc_pos += bytes;
- resp_range.consume(bytes);
- if (vio.ntodo() <= 0)
- return calluser(VC_EVENT_READ_COMPLETE);
- else {
- if (calluser(VC_EVENT_READ_READY) == EVENT_DONE)
- return EVENT_DONE;
- // we have to keep reading until we give the user all the
- // bytes it wanted or we hit the watermark.
- if (vio.ntodo() > 0 && !vio.buffer.writer()->high_water())
- goto Lread;
- return EVENT_CONT;
- }
-Lread: {
- if (vio.ndone >= (int64_t)doc_len)
- // reached the end of the document and the user still wants more
- return calluser(VC_EVENT_EOS);
- last_collision = 0;
- writer_lock_retry = 0;
- // if the state machine calls reenable on the callback from the cache,
- // we set up a schedule_imm event. The openReadReadDone discards
- // EVENT_IMMEDIATE events. So, we have to cancel that trigger and set
- // a new EVENT_INTERVAL event.
- cancel_trigger();
- CACHE_TRY_LOCK(lock, vol->mutex, mutex->thread_holding);
- if (!lock.is_locked()) {
- SET_HANDLER(&CacheVC::openReadMain);
- VC_SCHED_LOCK_RETRY();
- }
- if (dir_probe(&key, vol, &dir, &last_collision)) {
- SET_HANDLER(&CacheVC::openReadReadDone);
- int ret = do_read_call(&key);
- if (ret == EVENT_RETURN)
- goto Lcallreturn;
- return EVENT_CONT;
- } else if (write_vc) {
- if (writer_done()) {
- last_collision = NULL;
- while (dir_probe(&earliest_key, vol, &dir, &last_collision)) {
- if (dir_offset(&dir) == dir_offset(&earliest_dir)) {
- DDebug("cache_read_agg", "%p: key: %X ReadMain complete: %d",
- this, first_key.slice32(1), (int)vio.ndone);
- doc_len = vio.ndone;
- goto Leos;
- }
+ } else if (write_vc) {
+ if (writer_done()) {
+ last_collision = NULL;
+ while (dir_probe(&earliest_key, vol, &dir, &last_collision)) {
+ if (dir_offset(&dir) == dir_offset(&earliest_dir)) {
+ DDebug("cache_read_agg", "%p: key: %X ReadMain complete: %d",
+ this, first_key.slice32(1), (int)vio.ndone);
+ doc_len = vio.ndone;
+ goto Leos;
}
- DDebug("cache_read_agg", "%p: key: %X ReadMain writer aborted: %d",
- this, first_key.slice32(1), (int)vio.ndone);
- goto Lerror;
}
- DDebug("cache_read_agg", "%p: key: %X ReadMain retrying: %d", this, first_key.slice32(1), (int)vio.ndone);
- SET_HANDLER(&CacheVC::openReadMain);
- VC_SCHED_WRITER_RETRY();
+ DDebug("cache_read_agg", "%p: key: %X ReadMain writer aborted: %d",
+ this, first_key.slice32(1), (int)vio.ndone);
+ goto Lerror;
}
- if (is_action_tag_set("cache"))
- ink_release_assert(false);
- Warning("Document %X truncated at %d of %d, missing fragment %X", first_key.slice32(1), (int)vio.ndone, (int)doc_len, key.slice32(1));
- // remove the directory entry
- dir_delete(&earliest_key, vol, &earliest_dir);
+ DDebug("cache_read_agg", "%p: key: %X ReadMain retrying: %d", this, first_key.slice32(1), (int)vio.ndone);
+ SET_HANDLER(&CacheVC::openReadMain);
+ VC_SCHED_WRITER_RETRY();
}
+ if (is_action_tag_set("cache"))
+ ink_release_assert(false);
+ Warning("Document %X truncated at %d of %d, missing fragment %X", first_key.slice32(1), (int)vio.ndone, (int)doc_len, key.slice32(1));
+ // remove the directory entry
+ dir_delete(&earliest_key, vol, &earliest_dir);
Lerror:
return calluser(VC_EVENT_ERROR);
Leos:
@@ -867,6 +886,7 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS
doc_pos = doc->prefix_len();
next_CacheKey(&key, &doc->key);
fragment = 1;
+ frag_upper_bound = doc->data_len();
vol->begin_read(this);
if (vol->within_hit_evacuate_window(&earliest_dir) &&
(!cache_config_hit_evacuate_size_limit || doc_len <= (uint64_t)cache_config_hit_evacuate_size_limit)
@@ -1152,14 +1172,19 @@ CacheVC::openReadStartHead(int event, Event * e)
alternate.copy_shallow(alternate_tmp);
alternate.object_key_get(&key);
doc_len = alternate.object_size_get();
- resp_range.setRangeSpec(&(alternate.response_get()->getRangeSpec()));
- if (!resp_range.apply(request.getRangeSpec(), doc_len)) {
+
+ // Handle any range related setup.
+ if (!resp_range.init(&request)) {
+ err = ECACHE_INVALID_RANGE;
+ goto Ldone;
+ }
+ if (!resp_range.apply(doc_len)) {
err = ECACHE_UNSATISFIABLE_RANGE;
goto Ldone;
}
- resp_range.setContentType(alternate.response_get());
if (resp_range.isMulti())
- resp_range.generateBoundaryStr(earliest_key);
+ resp_range.setContentTypeFromResponse(alternate.response_get()).generateBoundaryStr(earliest_key);
+
if (key == doc->key) { // is this my data?
f.single_fragment = doc->single_fragment();
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/iocore/cache/I_Cache.h
----------------------------------------------------------------------
diff --git a/iocore/cache/I_Cache.h b/iocore/cache/I_Cache.h
index ffdf1ba..19e0592 100644
--- a/iocore/cache/I_Cache.h
+++ b/iocore/cache/I_Cache.h
@@ -58,6 +58,7 @@ class CacheLookupHttpConfig;
class URL;
class HTTPHdr;
class HTTPInfo;
+class HTTPRangeSpec;
typedef HTTPHdr CacheHTTPHdr;
typedef URL CacheURL;
@@ -231,6 +232,12 @@ struct CacheVConnection:public VConnection
This is the actual content sized modified (if appropriate) by range data.
*/
virtual uint64_t get_http_content_size() = 0;
+ /** Get the range spec for the response (request ranges modifed by content length).
+ */
+ virtual HTTPRangeSpec& get_http_range_spec() = 0;
+
+ /// Check if this is HTTP partial content (range request/response).
+ virtual bool is_http_partial_content() = 0;
#endif
virtual bool is_ram_cache_hit() const = 0;
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/iocore/cache/P_CacheHttp.h
----------------------------------------------------------------------
diff --git a/iocore/cache/P_CacheHttp.h b/iocore/cache/P_CacheHttp.h
index f76e842..91756b9 100644
--- a/iocore/cache/P_CacheHttp.h
+++ b/iocore/cache/P_CacheHttp.h
@@ -101,14 +101,10 @@ class CacheRange
typedef CacheRange self; ///< Self reference type.
/// Default constructor
- CacheRange() : _offset(0), _idx(-1), _r(NULL), _ct_field(NULL) { }
+ CacheRange() : _offset(0), _idx(-1), _ct_field(NULL), _pending_range_shift_p(false) { }
- /// Set the internal range spec pointer to @a src.
- self& setRangeSpec(HTTPRangeSpec* src);
-
- /// Test if the range should be active (used).
- /// @internal This means it has ranges and should be used to do seeks on the content.
- bool isActive() const;
+ /// Test if the range spec has actual ranges in it
+ bool hasRanges() const;
/// Test for multiple ranges.
bool isMulti() const;
@@ -119,6 +115,9 @@ class CacheRange
/// Get the current range index.
int getIdx() const;
+ /// Get the number of ranges.
+ size_t count() const;
+
/// Get the remaining contiguous bytes for the current range.
uint64_t getRemnantSize() const;
@@ -128,11 +127,15 @@ class CacheRange
*/
uint64_t consume(uint64_t size);
- /** Apply a @a src range and content @a len to the contained range spec.
+ /** Initialize from a request header.
+ */
+ bool init(HTTPHdr* req);
+
+ /** Apply a content @a len to the ranges.
@return @c true if successfully applied, @c false otherwise.
*/
- bool apply(HTTPRangeSpec const& src, uint64_t len);
+ bool apply(uint64_t len);
/** Get the range boundary string.
@a len if not @c NULL receives the length of the string.
@@ -140,25 +143,44 @@ class CacheRange
char const* getBoundaryStr(int* len) const;
/** Generate the range boundary string */
- void generateBoundaryStr(CacheKey const& key);
+ self& generateBoundaryStr(CacheKey const& key);
- /** Stash the Content-Type field pointer from a @a header.
+ /// Get the cached Content-Type field.
+ MIMEField* getContentTypeField() const;
- @return @c true if a Content-Type field was found in @a header, @c false if not.
- */
- bool setContentType(HTTPHdr* header);
+ /// Set the Content-Type field from a response header.
+ self& setContentTypeFromResponse(HTTPHdr* resp);
/** Calculate the effective HTTP content length value.
*/
uint64_t calcContentLength() const;
+ /// Raw access to internal range spec.
+ HTTPRangeSpec& getRangeSpec();
+
+ /// Test if a consume moved across a range boundary.
+ bool hasPendingRangeShift() const;
+
+ /// Clear the pending range shift flag.
+ self& consumeRangeShift();
+
+ /// Range access.
+ HTTPRangeSpec::Range& operator [] (int n);
+
+ /// Range access.
+ HTTPRangeSpec::Range const& operator [] (int n) const;
+
+ /// Reset to re-usable state.
+ void clear();
+
protected:
uint64_t _len; ///< Total object length.
uint64_t _offset; ///< Offset in content.
int _idx; ///< Current range index. (< 0 means not in a range)
- HTTPRangeSpec* _r; ///< The actual ranges.
+ HTTPRangeSpec _r; ///< The actual ranges.
MIMEField* _ct_field; ///< Content-Type field.
char _boundary[HTTP_RANGE_BOUNDARY_LEN];
+ bool _pending_range_shift_p;
};
TS_INLINE CacheHTTPInfo *
@@ -169,23 +191,10 @@ CacheHTTPInfoVector::get(int idx)
return &data[idx].alternate;
}
-inline CacheRange&
-CacheRange::setRangeSpec(HTTPRangeSpec* src)
-{
- _r = src;
- return *this;
-}
-
-inline bool
-CacheRange::apply(HTTPRangeSpec const& src, uint64_t len)
-{
- return _r && _r->apply(src, len);
-}
-
inline bool
-CacheRange::isActive() const
+CacheRange::hasRanges() const
{
- return _r && (_r->isSingle() || _r->isMulti());
+ return _r.isSingle() || _r.isMulti();
}
inline uint64_t
@@ -205,10 +214,10 @@ CacheRange::getRemnantSize() const
{
uint64_t zret = 0;
- if (!_r || _r->isEmpty())
+ if (_r.isEmpty())
zret = _len - _offset;
- else if (_r->isValid() && 0 <= _idx && _idx < static_cast<int>(_r->count()))
- zret = ((*_r)[_idx]._max - _offset) + 1;
+ else if (_r.isValid() && 0 <= _idx && _idx < static_cast<int>(_r.count()))
+ zret = (_r[_idx]._max - _offset) + 1;
return zret;
}
@@ -220,10 +229,60 @@ CacheRange::getBoundaryStr(int* len) const
return _boundary;
}
+inline HTTPRangeSpec&
+CacheRange::getRangeSpec()
+{
+ return _r;
+}
+
inline bool
CacheRange::isMulti() const
{
- return _r && _r->isMulti();
+ return _r.isMulti();
+}
+
+inline bool
+CacheRange::hasPendingRangeShift() const
+{
+ return _pending_range_shift_p;
+}
+
+inline CacheRange&
+CacheRange::consumeRangeShift()
+{
+ _pending_range_shift_p = false;
+ return *this;
+}
+
+inline MIMEField*
+CacheRange::getContentTypeField() const
+{
+ return _ct_field;
+}
+
+inline size_t
+CacheRange::count() const
+{
+ return _r.count();
+}
+
+inline HTTPRangeSpec::Range&
+CacheRange::operator [] (int n)
+{
+ return _r[n];
+}
+
+inline HTTPRangeSpec::Range const&
+CacheRange::operator [] (int n) const
+{
+ return _r[n];
+}
+
+inline CacheRange&
+CacheRange::setContentTypeFromResponse(HTTPHdr* resp)
+{
+ _ct_field = resp->field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
+ return *this;
}
#endif /* __CACHE_HTTP_H__ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/iocore/cache/P_CacheInternal.h
----------------------------------------------------------------------
diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h
index e386c98..c23112b 100644
--- a/iocore/cache/P_CacheInternal.h
+++ b/iocore/cache/P_CacheInternal.h
@@ -376,6 +376,8 @@ struct CacheVC: public CacheVConnection
virtual char const* get_http_range_boundary_string(int* len) const;
virtual uint64_t get_http_content_size();
+ virtual HTTPRangeSpec& get_http_range_spec();
+ virtual bool is_http_partial_content();
#endif
@@ -426,6 +428,7 @@ struct CacheVC: public CacheVConnection
#ifdef CACHE_STAT_PAGES
LINK(CacheVC, stat_link);
#endif
+ CacheRange resp_range; ///< Tracking information for range data for response.
// end Region B
// Start Region C
@@ -467,7 +470,6 @@ struct CacheVC: public CacheVConnection
uint64_t total_len; // total length written and available to write
uint64_t doc_len; // total_length (of the selected alternate for HTTP)
uint64_t update_len;
- CacheRange resp_range; ///< Tracking information for range data for response.
/// The offset in the content of the first byte beyond the end of the current fragment.
/// @internal This seems very weird but I couldn't figure out how to keep the more sensible
/// lower bound correctly updated.
@@ -648,6 +650,7 @@ free_CacheVC(CacheVC *cont)
cont->alternate_index = CACHE_ALT_INDEX_DEFAULT;
if (cont->scan_vol_map)
ats_free(cont->scan_vol_map);
+ cont->resp_range.clear();
memset((char *) &cont->vio, 0, cont->size_to_init);
#ifdef CACHE_STAT_PAGES
ink_assert(!cont->stat_link.next && !cont->stat_link.prev);
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/iocore/cluster/P_ClusterCache.h
----------------------------------------------------------------------
diff --git a/iocore/cluster/P_ClusterCache.h b/iocore/cluster/P_ClusterCache.h
index c7a3033..8325f07 100644
--- a/iocore/cluster/P_ClusterCache.h
+++ b/iocore/cluster/P_ClusterCache.h
@@ -358,6 +358,8 @@ struct ClusterVConnectionBase: public CacheVConnection
// link problems so I can get the base case to work.
virtual char const* get_http_range_boundary_string(int*) const { return NULL; }
virtual uint64_t get_http_content_size() { return this->get_object_size(); }
+ virtual HTTPRangeSpec& get_http_range_spec() { return resp_range.getRangeSpec(); }
+ virtual bool is_http_partial_content() { return false; }
// Set the timeouts associated with this connection.
// active_timeout is for the total elasped time of the connection.
@@ -390,6 +392,7 @@ struct ClusterVConnectionBase: public CacheVConnection
ink_hrtime active_timeout_in;
Event *inactivity_timeout;
Event *active_timeout;
+ CacheRange resp_range;
virtual void reenable(VIO *);
virtual void reenable_re(VIO *);
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/lib/ts/InkErrno.h
----------------------------------------------------------------------
diff --git a/lib/ts/InkErrno.h b/lib/ts/InkErrno.h
index 45a846d..14c17ea 100644
--- a/lib/ts/InkErrno.h
+++ b/lib/ts/InkErrno.h
@@ -66,7 +66,8 @@
#define ECACHE_NOT_READY (CACHE_ERRNO+7)
#define ECACHE_ALT_MISS (CACHE_ERRNO+8)
#define ECACHE_BAD_READ_REQUEST (CACHE_ERRNO+9)
-#define ECACHE_UNSATISFIABLE_RANGE (CACHE_ERRNO+10)
+#define ECACHE_INVALID_RANGE (CACHE_ERRNO+10)
+#define ECACHE_UNSATISFIABLE_RANGE (CACHE_ERRNO+11)
#define EHTTP_ERROR (HTTP_ERRNO+0)
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/proxy/hdrs/HTTP.cc
----------------------------------------------------------------------
diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc
index d2d5f69..58beb4f 100644
--- a/proxy/hdrs/HTTP.cc
+++ b/proxy/hdrs/HTTP.cc
@@ -29,6 +29,7 @@
#include "HTTP.h"
#include "HdrToken.h"
#include "Diags.h"
+#include "I_IOBuffer.h"
/***********************************************************************
* *
@@ -2151,7 +2152,7 @@ HTTPRangeSpec::parse(char const* v, int len)
src.skip(&ParseRules::is_ws);
if (src.size() > sizeof(HTTP_LEN_BYTES)+1 &&
- 0 == memcmp(src.data(), HTTP_VALUE_BYTES, HTTP_LEN_BYTES) && '=' == src[HTTP_LEN_BYTES]
+ 0 == strncasecmp(src.data(), HTTP_VALUE_BYTES, HTTP_LEN_BYTES) && '=' == src[HTTP_LEN_BYTES]
) {
_state = INVALID; // something, it needs to be correct.
src += HTTP_LEN_BYTES+1;
@@ -2226,10 +2227,55 @@ HTTPRangeSpec::add(uint64_t low, uint64_t high)
}
bool
+HTTPRangeSpec::apply(uint64_t len)
+{
+ if (!this->hasRanges()) {
+ // nothing - simplifying later logic.
+ } else if (0 == len) {
+ /* Must special case zero length content
+ - suffix ranges are OK but other ranges are not.
+ - Best option is to return a 200 (not 206 or 416) for all suffix range spec on zero length content.
+ (this is what Apache HTTPD does)
+ - So, mark result as either @c UNSATISFIABLE or @c EMPTY, clear all ranges.
+ */
+ _state = EMPTY;
+ if (!_single.isSuffix()) _state = UNSATISFIABLE;
+ for ( RangeBox::iterator spot = _ranges.begin(), limit = _ranges.end() ; spot != limit && EMPTY == _state ; ++spot ) {
+ if (!spot->isSuffix()) _state = UNSATISFIABLE;
+ }
+ _ranges.clear();
+ } else if (this->isSingle()) {
+ if (!_single.apply(len)) _state = UNSATISFIABLE;
+ } else { // gotta be MULTI
+ int src = 0, dst = 0;
+ int n = _ranges.size();
+ while (src < n) {
+ Range& r = _ranges[src];
+ if (r.apply(len)) {
+ if (src != dst) _ranges[dst] = r;
+ ++dst;
+ }
+ ++src;
+ }
+ // at this point, @a dst is the # of valid ranges.
+ if (dst > 0) {
+ _single = _ranges[0];
+ if (dst == 1) _state = SINGLE;
+ _ranges.resize(dst);
+ } else {
+ _state = UNSATISFIABLE;
+ _ranges.clear();
+ }
+ }
+ return this->isValid();
+}
+
+# if 0
+bool
HTTPRangeSpec::apply(self const& that, uint64_t len)
{
_state = that._state;
- if (INVALID == _state || EMPTY == _state) {
+ if (INVALID == _state || EMPTY == _state || UNSATISFIABLE == _state) {
// nothing - simplifying later logic.
} else if (0 == len) {
/* Must special case zero length content
@@ -2278,12 +2324,27 @@ HTTPHdr::getRangeSpec()
m_range_parsed = true;
return m_range_spec;
}
+# endif
-int
-Calc_Digital_Length(uint64_t x)
+namespace {
+
+ int
+ Calc_Digital_Length(uint64_t x)
+ {
+ char buff[32]; // big enough for 64 bit #
+ return snprintf(buff, sizeof(buff), "%" PRIu64, x);
+ }
+
+}
+
+uint64_t
+HTTPRangeSpec::calcPartBoundarySize(uint64_t object_size, uint64_t ct_val_len)
{
- char buff[32]; // big enough for 64 bit #
- return snprintf(buff, sizeof(buff), "%" PRIu64, x);
+ size_t l_size = Calc_Digital_Length(object_size);
+ // CR LF "--" boundary-string CR LF "Content-Range" ": " "bytes " X "-" Y "/" Z CR LF Content-Type CR LF
+ uint64_t zret = 4 + HTTP_RANGE_BOUNDARY_LEN + 2 + MIME_LEN_CONTENT_RANGE + 2 + HTTP_LEN_BYTES + 1 + l_size + 1 +l_size + 1 + l_size + 2;
+ if (ct_val_len) zret += MIME_LEN_CONTENT_TYPE + 2 + ct_val_len + 2;
+ return zret;
}
uint64_t
@@ -2293,16 +2354,62 @@ HTTPRangeSpec::calcContentLength(uint64_t object_size, uint64_t ct_val_len) cons
size_t nr = this->count();
if (nr >= 1) {
- size = this->size();
- if (nr > 1) {
- size_t l_size = Calc_Digital_Length(object_size);
- // CR LF "--" boundary-string CR LF "Content-Range" ": " "bytes " X "-" Y "/" Z
- uint64_t sep_size = 2 + 2 + HTTP_RANGE_BOUNDARY_LEN + 2 + MIME_LEN_CONTENT_RANGE + 2 + HTTP_LEN_BYTES + 1 + l_size + 1 +l_size + 1 + l_size + 2;
-
- sep_size += MIME_LEN_CONTENT_TYPE + 2 + ct_val_len + 2;
- size += nr * sep_size;
- }
+ size = this->size(); // the real content size.
+ if (nr > 1) // part boundaries
+ size += nr * self::calcPartBoundarySize(object_size, ct_val_len) + 2; // need trailing '--'
}
return size;
}
+uint64_t
+HTTPRangeSpec::writePartBoundary(MIOBuffer* out, char const* boundary_str, size_t boundary_len, uint64_t total_size, uint64_t low, uint64_t high, MIMEField* ctf, bool final)
+{
+ size_t x; // tmp for printf results.
+ size_t loc_size = Calc_Digital_Length(total_size)*3 + 3; // precomputed size of all the location / size text.
+ size_t n = self::calcPartBoundarySize(total_size, ctf ? ctf->m_len_value : 0) + (final ? 2 : 0);
+ Ptr<IOBufferData> d(new_IOBufferData(iobuffer_size_to_index(n, MAX_BUFFER_SIZE_INDEX), MEMALIGNED));
+ char* spot = d->data();
+
+ x = snprintf(spot, n, "\r\n--%.*s", static_cast<int>(boundary_len), boundary_str);
+ spot += x;
+ n -= x;
+ if (final) {
+ memcpy(spot, "--", 2);
+ spot += 2;
+ n -= 2;
+ }
+
+ x = snprintf(spot, n, "\r\n%.*s: %.*s", MIME_LEN_CONTENT_RANGE, MIME_FIELD_CONTENT_RANGE, HTTP_LEN_BYTES, HTTP_VALUE_BYTES);
+ spot += x;
+ n -= x;
+ spot[-HTTP_LEN_BYTES] = tolower(spot[-HTTP_LEN_BYTES]); // ugly cleanup just to be careful of stupid user agents.
+
+ x = snprintf(spot, n, " %" PRIu64 "-%" PRIu64 "/%" PRIu64, low, high, total_size);
+ // Need to space fill to match pre-computed size
+ if (x < loc_size) memset(spot+x, ' ', loc_size - x);
+ spot += loc_size;
+ n -= loc_size;
+
+ if (ctf) {
+ int ctf_len;
+ char const* ctf_val = ctf->value_get(&ctf_len);
+ if (ctf_val) {
+ x = snprintf(spot, n, "\r\n%.*s: %.*s", MIME_LEN_CONTENT_TYPE, MIME_FIELD_CONTENT_TYPE, ctf_len, ctf_val);
+ spot += x;
+ n -= x;
+ }
+ }
+
+ // This also takes care of the snprintf null termination problem.
+ *spot++ = '\r';
+ *spot++ = '\n';
+ n -= 2;
+
+ ink_assert(n == 0);
+
+ IOBufferBlock* b = new_IOBufferBlock(d, spot - d->data());
+ b->_buf_end = b->_end;
+ out->append_block(b);
+
+ return spot - d->data();
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/proxy/hdrs/HTTP.h
----------------------------------------------------------------------
diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h
index 559a3b2..91316e2 100644
--- a/proxy/hdrs/HTTP.h
+++ b/proxy/hdrs/HTTP.h
@@ -38,6 +38,7 @@
#define HTTP_MAJOR(v) (((v) >> 16) & 0xFFFF)
class Http2HeaderTable;
+class MIOBuffer;
enum HTTPStatus
{
@@ -578,11 +579,15 @@ struct HTTPRangeSpec {
/// Default constructor - invalid range
HTTPRangeSpec();
+ /// Reset to re-usable state.
+ void clear();
+
/** Parse a range field @a value and update @a this with the results.
@return @c true if @a value was a valid range specifier, @c false otherwise.
*/
bool parse(char const* value, int len);
+# if 0
/** Copy ranges from @a while applying them to the content @a length.
Ranges are copied if valid for @a length and converted to absolute offsets. The number of ranges
@@ -594,6 +599,18 @@ struct HTTPRangeSpec {
always satisfiable.
*/
bool apply(self const& that, uint64_t length);
+# endif
+
+ /** Update ranges to be absolute based on content @a length.
+
+ Invalid ranges are removed, ranges will be clipped as needed, and suffix ranges will be
+ converted to absolute ranges.
+
+ @return @c true if the range spec is satisfiable (there remains at least one valid range), @c false otherwise.
+ Note a range spec with no ranges is always satisfiable and that suffix ranges are also
+ always satisfiable.
+ */
+ bool apply(uint64_t length);
/** Number of distinct ranges.
@return Number of ranges.
@@ -624,6 +641,9 @@ struct HTTPRangeSpec {
/// Access the range at index @a idx.
Range& operator [] (int n);
+ /// Access the range at index @a idx.
+ Range const& operator [] (int n) const;
+
/** Calculate the content length for this range specification.
@note If a specific content length has not been @c apply 'd this will not produce
@@ -636,6 +656,25 @@ struct HTTPRangeSpec {
uint64_t ct_val_len ///< Length of Content-Type field value.
) const;
+ /// Calculate the length of the range part boundary header.
+ static uint64_t calcPartBoundarySize(
+ uint64_t object_size ///< Base content size
+ , uint64_t ct_val_len ///< Length of the Content-Type value (0 if none).
+ );
+
+ /** Write the range part boundary to @a out.
+ */
+ static uint64_t writePartBoundary(
+ MIOBuffer* out ///< Output IO Buffer
+ , char const* boundary_str ///< Boundary marker string.
+ , size_t boundary_len ///< Length of boundary marker string.
+ , uint64_t total_size ///< Base content size.
+ , uint64_t low ///< Low value for the range.
+ , uint64_t high ///< High value for the raNGE.
+ , MIMEField* ctf ///< Content-Type field (@c NULL if none)
+ , bool final ///< Is this the final part boundary?
+ );
+
/// Iterator for first range.
iterator begin();
/// Iterator past last range.
@@ -664,13 +703,6 @@ public:
/// also had a port, @c false otherwise.
mutable bool m_port_in_header;
- /// Parsed data from the RANGE field.
- /// For requests, this is the RANGE field specification.
- /// For responses, this is the RANGE field applied to the content length.
- HTTPRangeSpec m_range_spec;
- /// Have we parsed the range field yet?
- bool m_range_parsed;
-
HTTPHdr();
~HTTPHdr();
@@ -793,15 +825,6 @@ public:
const char *reason_get(int *length);
void reason_set(const char *value, int length);
- /// Get the internal @c HTTPRangeSpec instance.
- HTTPRangeSpec& getRangeSpec();
- /// Check if this response is a partial content response.
- /// @return @c true if there is a least one range, @c false if not.
- bool isPartialContent() const;
- /// Locate and parse (if present) the @c Range header field.
- /// The results are put in to the internal @c HTTPRangeSpec instance.
- bool parse_range();
-
MIMEParseResult parse_req(HTTPParser *parser, const char **start, const char *end, bool eof);
MIMEParseResult parse_resp(HTTPParser *parser, const char **start, const char *end, bool eof);
@@ -950,7 +973,7 @@ HTTPVersion::operator <=(const HTTPVersion & hv) const
inline
HTTPHdr::HTTPHdr()
- : MIMEHdr(), m_http(NULL), m_url_cached(), m_target_cached(false), m_range_parsed(false)
+ : MIMEHdr(), m_http(NULL), m_url_cached(), m_target_cached(false)
{ }
@@ -1679,6 +1702,13 @@ HTTPRangeSpec::HTTPRangeSpec() : _state(EMPTY)
{
}
+inline void
+HTTPRangeSpec::clear()
+{
+ _state = EMPTY;
+ RangeBox().swap(_ranges); // force memory drop.
+}
+
inline bool
HTTPRangeSpec::isSingle() const
{
@@ -1768,7 +1798,7 @@ HTTPRangeSpec::Range::apply(uint64_t len)
_max = len - 1;
_min = _min > len ? 0 : len - _min;
} else if (_min < len) {
- _max = MIN(_max,len);
+ _max = MIN(_max,len-1);
} else {
this->invalidate();
zret = false;
@@ -1782,6 +1812,12 @@ HTTPRangeSpec::operator [] (int n)
return SINGLE == _state ? _single : _ranges[n];
}
+inline HTTPRangeSpec::Range const&
+HTTPRangeSpec::operator [] (int n) const
+{
+ return SINGLE == _state ? _single : _ranges[n];
+}
+
inline HTTPRangeSpec::iterator
HTTPRangeSpec::begin()
{
@@ -1802,10 +1838,4 @@ HTTPRangeSpec::end()
}
}
-inline bool
-HTTPHdr::isPartialContent() const
-{
- return m_range_spec.hasRanges();
-}
-
#endif /* __HTTP_H__ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/proxy/http/HttpSM.cc
----------------------------------------------------------------------
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
index d1edc50..bb2b3fc 100644
--- a/proxy/http/HttpSM.cc
+++ b/proxy/http/HttpSM.cc
@@ -5812,7 +5812,8 @@ HttpSM::setup_cache_read_transfer()
ink_assert(cache_sm.cache_read_vc != NULL);
- doc_size = t_state.cache_info.object_read->object_size_get();
+// doc_size = t_state.cache_info.object_read->object_size_get();
+ doc_size = cache_sm.cache_read_vc->get_http_content_size();
alloc_index = buffer_size_to_index(doc_size + index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX));
#ifndef USE_NEW_EMPTY_MIOBUFFER
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/2ddfc7ab/proxy/http/HttpTransact.cc
----------------------------------------------------------------------
diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc
index 169e9ce..07c28d9 100644
--- a/proxy/http/HttpTransact.cc
+++ b/proxy/http/HttpTransact.cc
@@ -6585,7 +6585,7 @@ HttpTransact::handle_content_length_header(State* s, HTTPHdr* header, HTTPHdr* b
case SOURCE_CACHE:
// if we are doing a single Range: request, calculate the new
// C-L: header
- if (base->isPartialContent()) {
+ if (s->state_machine->get_cache_sm().cache_read_vc->is_http_partial_content()) {
change_response_header_because_of_range_request(s,header);
s->hdr_info.trust_response_cl = true;
}
@@ -8762,7 +8762,7 @@ HttpTransact::change_response_header_because_of_range_request(State *s, HTTPHdr
char *reason_phrase;
CacheVConnection* cache_read_vc = s->state_machine->get_cache_sm().cache_read_vc;
HTTPHdr* cached_response = find_appropriate_cached_resp(s);
- HTTPRangeSpec& rs = cached_response->getRangeSpec();
+ HTTPRangeSpec& rs = cache_read_vc->get_http_range_spec();
Debug("http_trans", "Partial content requested, re-calculating content-length");
@@ -8794,6 +8794,7 @@ HttpTransact::change_response_header_because_of_range_request(State *s, HTTPHdr
, rs[0]._min, rs[0]._max
, cached_response->get_content_length()
);
+// buff[0] = tolower(buff[0]);
field->value_set(header->m_heap, header->m_mime, buff, n);
header->field_attach(field);
}
[6/8] trafficserver git commit: Checkpoint before moving stuff to
HTTPCacheAlt.
Posted by am...@apache.org.
Checkpoint before moving stuff to HTTPCacheAlt.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/7b1f37d9
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/7b1f37d9
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/7b1f37d9
Branch: refs/heads/ts-974-multi-range-read
Commit: 7b1f37d937ced48d4699c91af6363bfa4ab69d80
Parents: ae369ee
Author: Alan M. Carroll <so...@yahoo-inc.com>
Authored: Thu Dec 4 17:11:25 2014 -0600
Committer: Alan M. Carroll <so...@yahoo-inc.com>
Committed: Sat Dec 6 11:56:04 2014 -0600
----------------------------------------------------------------------
iocore/cache/CacheHttp.cc | 162 ++++-----------------
iocore/cache/CacheRead.cc | 48 ++++---
iocore/cache/I_Cache.h | 11 ++
iocore/cache/I_CacheDefs.h | 158 --------------------
iocore/cache/P_CacheHttp.h | 132 ++++++++++++++++-
iocore/cache/P_CacheInternal.h | 77 +---------
lib/ts/InkErrno.h | 2 +-
proxy/hdrs/HTTP.cc | 174 ++++++++++++++++++++++
proxy/hdrs/HTTP.h | 280 +++++++++++++++++++++++++++++++++++-
proxy/http/HttpTransact.cc | 54 ++++---
10 files changed, 680 insertions(+), 418 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/iocore/cache/CacheHttp.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheHttp.cc b/iocore/cache/CacheHttp.cc
index 3b86a9b..4d82c18 100644
--- a/iocore/cache/CacheHttp.cc
+++ b/iocore/cache/CacheHttp.cc
@@ -258,157 +258,47 @@ CacheHTTPInfoVector::get_handles(const char *buf, int length, RefCountObj * bloc
/*-------------------------------------------------------------------------
-------------------------------------------------------------------------*/
-bool
-RangeSpec::parse(char const* v, int len)
-{
- // Maximum # of digits permitted for an offset. Avoid issues with overflow.
- static size_t const MAX_DIGITS = 15;
- char const PREFIX[] = { 'b', 'y', 't', 'e', 's', '=' };
- ts::ConstBuffer src(v, len);
- size_t n;
-
- _state = EMPTY;
- src.skip(&ParseRules::is_ws);
-
- if (src.size() > sizeof(PREFIX) && 0 == memcmp(src.data(), PREFIX, sizeof(PREFIX))) {
- _state = INVALID; // something, it needs to be correct.
- src += sizeof(PREFIX);
- while (src) {
- ts::ConstBuffer max = src.splitOn(',');
-
- if (!max) { // no comma so everything in @a src should be processed as a single range.
- max = src;
- src.reset();
- }
- ts::ConstBuffer min = max.splitOn('-');
-
- src.skip(&ParseRules::is_ws);
- // Spec forbids whitspace anywhere in the range element.
-
- if (min) {
- if (ParseRules::is_digit(*min) && min.size() <= MAX_DIGITS) {
- uint64_t low = ats_strto64(min.data(), min.size(), &n);
- if (n < min.size()) break; // extra cruft in range, not even ws allowed
- if (max) {
- if (ParseRules::is_digit(*max) && max.size() <= MAX_DIGITS) {
- uint64_t high = ats_strto64(max.data(), max.size(), &n);
- if (n < max.size() && (max += n).skip(&ParseRules::is_ws))
- break; // non-ws cruft after maximum
- else
- this->add(low, high);
- } else {
- break; // invalid characters for maximum
- }
- } else {
- this->add(low, UINT64_MAX); // "X-" : "offset X to end of content"
- }
- } else {
- break; // invalid characters for minimum
- }
- } else {
- if (max) {
- if (ParseRules::is_digit(*max) && max.size() <= MAX_DIGITS) {
- uint64_t high = ats_strto64(max.data(), max.size(), &n);
- if (n < max.size() && (max += n).skip(&ParseRules::is_ws)) {
- break; // cruft after end of maximum
- } else {
- this->add(high, 0);
- }
- } else {
- break; // invalid maximum
- }
- }
- }
+uint64_t
+CacheRange::consume(uint64_t size)
+{
+ switch (_r->_state) {
+ case HTTPRangeSpec::EMPTY: _offset += size; break;
+ case HTTPRangeSpec::SINGLE: _offset += std::min(size, (_r->_single._max - _offset) + 1 ); break;
+ case HTTPRangeSpec::MULTI:
+ while (size && _idx < static_cast<int>(_r->count())) {
+ uint64_t r = std::min(size, ((*_r)[_idx]._max - _offset) + 1);
+ _offset += r;
+ size -= r;
+ if (_offset > (*_r)[_idx]._max)
+ _offset = (*_r)[++_idx]._min;
}
- if (src) _state = INVALID; // didn't parse everything, must have been an error.
+ break;
+ default: break;
}
- return _state != INVALID;
-}
-RangeSpec&
-RangeSpec::add(uint64_t low, uint64_t high)
-{
- if (MULTI == _state) {
- _ranges.push_back(Range(low, high));
- } else if (SINGLE == _state) {
- _ranges.push_back(_single);
- _ranges.push_back(Range(low,high));
- _state = MULTI;
- } else {
- _single._min = low;
- _single._max = high;
- _state = SINGLE;
- }
- return *this;
+ return _offset;
}
-bool
-RangeSpec::finalize(uint64_t len)
+void
+CacheRange::generateBoundaryStr(CacheKey const& key)
{
- if (INVALID == _state || EMPTY == _state) {
- // nothing but simplifying later logic.
- } else if (0 == len) {
- /* Must special case zero length content
- - suffix ranges are OK but other ranges are not.
- - SM must return a 200 (not 206 or 416) for a valid range on zero length content.
- (this is what Apache HTTPD does and seems the least bad thing)
- - Therefore we don't bother actually adjusting the ranges as values don't matter.
- */
- if (!_single.isSuffix()) _state = INVALID;
- if (MULTI == _state) {
- for ( RangeBox::iterator spot = _ranges.begin(), limit = _ranges.end() ; spot != limit && MULTI == _state ; ++spot ) {
- if (!spot->isSuffix()) _state = INVALID;
- }
- }
- } else { // len > 0
- if (!_single.finalize(len)) _state = INVALID;
- if (MULTI == _state) {
- for ( RangeBox::iterator spot = _ranges.begin(), limit = _ranges.end() ; spot != limit && MULTI == _state; ++spot ) {
- if (!spot->finalize(len)) _state = INVALID;
- }
- }
- }
- return INVALID != _state;
+ snprintf(_boundary, sizeof(_boundary), "%08" PRIu64 "%08" PRIu64 "..%08" PRIu64
+ , key.slice64(0), key.slice64(1), this_ethread()->generator.random()
+ );
}
-/*-------------------------------------------------------------------------
- -------------------------------------------------------------------------*/
bool
-CacheRange::finalize(uint64_t len)
+CacheRange::setContentType(HTTPHdr* header)
{
- bool zret = super::finalize(len);
- if (zret) {
- if (this->isEmpty()) { // pretend it's one range [0..len)
- _offset = 0;
- } else {
- _idx = 0;
- _offset = _single._min;
- }
- _len = len;
- }
- return zret;
+ _ct_field = header->field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
+ return NULL != _ct_field;
}
uint64_t
-CacheRange::consume(uint64_t size)
+CacheRange::calcContentLength() const
{
- switch (_state) {
- case EMPTY: _offset += size; break;
- case SINGLE: _offset += std::min(size, (_single._max - _offset) + 1 ); break;
- case MULTI:
- while (size && _idx < static_cast<int>(_ranges.size())) {
- uint64_t r = std::min(size, (_ranges[_idx]._max - _offset) + 1);
- _offset += r;
- size -= r;
- if (_offset > _ranges[_idx]._max)
- _offset = _ranges[++_idx]._min;
- }
- break;
- default: break;
- }
-
- return _offset;
+ return _r->calcContentLength(_len, _ct_field->m_len_value);
}
/*-------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/iocore/cache/CacheRead.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc
index f263d82..ac5e13e 100644
--- a/iocore/cache/CacheRead.cc
+++ b/iocore/cache/CacheRead.cc
@@ -109,8 +109,6 @@ Cache::open_read(Continuation * cont, CacheKey * key, CacheHTTPHdr * request,
{
CACHE_TRY_LOCK(lock, vol->mutex, mutex->thread_holding);
if (!lock.is_locked() || (od = vol->open_read(key)) || dir_probe(key, vol, &result, &last_collision)) {
- MIMEField* range_field = request->field_find(MIME_FIELD_RANGE, MIME_LEN_RANGE);
-
c = new_CacheVC(cont);
c->first_key = c->key = c->earliest_key = *key;
c->vol = vol;
@@ -121,12 +119,6 @@ Cache::open_read(Continuation * cont, CacheKey * key, CacheHTTPHdr * request,
c->frag_type = CACHE_FRAG_TYPE_HTTP;
c->params = params;
c->od = od;
- if (range_field) {
- char const* value;
- int len;
- value = range_field->value_get(&len);
- c->req_rs.parse(value, len);
- }
}
if (!lock.is_locked()) {
SET_CONTINUATION_HANDLER(c, &CacheVC::openReadStartHead);
@@ -180,6 +172,18 @@ CacheVC::load_http_info(CacheHTTPInfoVector* info, Doc* doc, RefCountObj * block
return zret;
}
+char const*
+CacheVC::get_http_range_boundary_string(int* len) const
+{
+ return resp_range.getBoundaryStr(len);
+}
+
+uint64_t
+CacheVC::get_http_content_size()
+{
+ return resp_range.calcContentLength();
+}
+
int
CacheVC::openReadFromWriterFailure(int event, Event * e)
{
@@ -636,9 +640,7 @@ Lcallreturn:
LreadMain:
++fragment;
doc_pos = doc->prefix_len();
- if (req_rs.isValid()) {
- doc_pos += req_rs.getOffset() - frag_upper_bound; // used before update!
- }
+ doc_pos += resp_range.getOffset() - frag_upper_bound; // used before update!
frag_upper_bound += doc->data_len();
next_CacheKey(&key, &key);
SET_HANDLER(&CacheVC::openReadMain);
@@ -692,9 +694,9 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
int64_t bytes = doc->len - doc_pos;
IOBufferBlock *b = NULL;
#ifdef HTTP_CACHE
- if (req_rs.isValid()) {
+ if (resp_range.isActive()) {
int target = -1; // target fragment index.
- uint64_t target_offset = req_rs.getOffset();
+ uint64_t target_offset = resp_range.getOffset();
uint64_t lower_bound = frag_upper_bound - doc->data_len();
if (target_offset < lower_bound || frag_upper_bound <= target_offset) {
@@ -703,7 +705,7 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
if (is_debug_tag_set("amc")) {
char b[33], c[33];
Debug("amc", "Seek @ %" PRIu64 " [r#=%d] in %s from #%d @ %" PRIu64 "/%d/%" PRId64 ":%s%s",
- target_offset, req_rs.getIdx(), first_key.toHexStr(b), fragment, frag_upper_bound, doc->len, doc->total_len, doc->key.toHexStr(c)
+ target_offset, resp_range.getIdx(), first_key.toHexStr(b), fragment, frag_upper_bound, doc->len, doc->total_len, doc->key.toHexStr(c)
, (frags ? "" : "no frag table")
);
}
@@ -725,7 +727,7 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
doc_pos = r_doc_pos;
bytes = doc->len - doc_pos;
}
- bytes = std::min(bytes, static_cast<int64_t>(req_rs.getRemnantSize()));
+ bytes = std::min(bytes, static_cast<int64_t>(resp_range.getRemnantSize()));
}
#endif
}
@@ -743,7 +745,7 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
vio.buffer.writer()->append_block(b);
vio.ndone += bytes;
doc_pos += bytes;
- req_rs.consume(bytes);
+ resp_range.consume(bytes);
if (vio.ntodo() <= 0)
return calluser(VC_EVENT_READ_COMPLETE);
else {
@@ -1150,6 +1152,15 @@ CacheVC::openReadStartHead(int event, Event * e)
alternate.copy_shallow(alternate_tmp);
alternate.object_key_get(&key);
doc_len = alternate.object_size_get();
+ resp_range.setRangeSpec(&(alternate.response_get()->getRangeSpec()));
+ if (!resp_range.apply(request.getRangeSpec(), doc_len)) {
+ err = ECACHE_UNSATISFIABLE_RANGE;
+ goto Ldone;
+ }
+ resp_range.setContentType(alternate.response_get());
+ if (resp_range.isMulti())
+ resp_range.generateBoundaryStr(earliest_key);
+
if (key == doc->key) { // is this my data?
f.single_fragment = doc->single_fragment();
ink_assert(f.single_fragment); // otherwise need to read earliest
@@ -1172,11 +1183,6 @@ CacheVC::openReadStartHead(int event, Event * e)
doc_len = doc->total_len;
}
- if (!req_rs.finalize(doc_len)) {
- err = ECACHE_BAD_REQUEST_RANGE;
- goto Ldone;
- }
-
if (is_debug_tag_set("cache_read")) { // amc debug
char xt[33],yt[33];
Debug("cache_read", "CacheReadStartHead - read %s target %s - %s %d of %" PRId64" bytes, %d fragments",
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/iocore/cache/I_Cache.h
----------------------------------------------------------------------
diff --git a/iocore/cache/I_Cache.h b/iocore/cache/I_Cache.h
index 1606a23..ffdf1ba 100644
--- a/iocore/cache/I_Cache.h
+++ b/iocore/cache/I_Cache.h
@@ -220,6 +220,17 @@ struct CacheVConnection:public VConnection
#ifdef HTTP_CACHE
virtual void set_http_info(CacheHTTPInfo *info) = 0;
virtual void get_http_info(CacheHTTPInfo **info) = 0;
+
+ /** Get the boundary string for a multi-part range response.
+ The length of the string is returned in @a len.
+
+ @return A point to the string.
+ */
+ virtual char const* get_http_range_boundary_string(int* len) const = 0;
+ /** Get the effective content size.
+ This is the actual content sized modified (if appropriate) by range data.
+ */
+ virtual uint64_t get_http_content_size() = 0;
#endif
virtual bool is_ram_cache_hit() const = 0;
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/iocore/cache/I_CacheDefs.h
----------------------------------------------------------------------
diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h
index ce50958..6d8392c 100644
--- a/iocore/cache/I_CacheDefs.h
+++ b/iocore/cache/I_CacheDefs.h
@@ -132,162 +132,4 @@ typedef CryptoHash CacheKey;
word(3) - ram cache hash, lookaside cache
*/
-
-/** A range specification.
-
- This represents the data for an HTTP range specification.
-*/
-struct RangeSpec {
- typedef RangeSpec self;
-
- /** A range of bytes in an object.
-
- If @a _min > 0 and @a _max == 0 the range is backwards and counts from the
- end of the object. That is (100,0) means the last 100 bytes of content.
- */
- struct Range {
- uint64_t _min;
- uint64_t _max;
-
- /// Default constructor - invalid range.
- Range() : _min(UINT64_MAX), _max(1) { }
- /// Construct as the range ( @a low .. @a high )
- Range(uint64_t low, uint64_t high) : _min(low), _max(high) {}
-
- /// Test if this range is a trailing (terminal) range.
- bool isSuffix() const;
- /// Test if this range is a valid range.
- bool isValid() const;
- /// Adjust the range values based on content size @a len.
- bool finalize(uint64_t len);
- /// Force the range to an invalid state.
- Range& invalidate();
- };
-
- /// Current state of the overall specification.
- /// @internal We can distinguish between @c SINGLE and @c MULTI by looking at the
- /// size of @a _ranges but we need this to mark @c EMPTY vs. not.
- enum State {
- EMPTY, ///< No range.
- INVALID, ///< Range parsing failed.
- SINGLE, ///< Single range.
- MULTI, ///< Multiple ranges.
- } _state;
-
- /// The first range value.
- /// By separating this out we can avoid allocation in the case of a single
- /// range value, which is by far the most common ( > 99% in my experience).
- Range _single;
- /// Storage for range values.
- typedef std::vector<Range> RangeBox;
- /// The first range is copied here if there is more than one (to simplify).
- RangeBox _ranges;
-
- /// Default constructor - invalid range
- RangeSpec();
-
- /** Parse a range field and update @a this with the results.
- @return @c true if @a v was a valid range specifier, @c false otherwise.
- */
- bool parse(char const* v, int len);
-
- /** Validate and convert for a specific content @a length.
-
- @return @c true if the range is satisfiable per the HTTP spec, @c false otherwise.
- Note a range spec with no ranges is always satisfiable.
- */
- bool finalize(uint64_t length);
-
- /** Number of distinct ranges.
- @return Number of ranges.
- */
- size_t count() const;
-
- /// If this is a valid single range specification.
- bool isSingle() const;
-
- /// If this is a valid multi range specification.
- bool isMulti() const;
-
- /// Test if this contains at least one valid range.
- bool isValid() const;
-
- /// Test if this is a valid but empty range spec.
- bool isEmpty() const;
-
-protected:
- self& add(uint64_t low, uint64_t high);
-};
-
-inline
-RangeSpec::RangeSpec() : _state(EMPTY)
-{
-}
-
-inline bool
-RangeSpec::isSingle() const
-{
- return SINGLE == _state;
-}
-
-inline bool
-RangeSpec::isMulti() const
-{
- return MULTI == _state;
-}
-
-inline bool
-RangeSpec::isEmpty() const
-{
- return EMPTY == _state;
-}
-
-inline size_t
-RangeSpec::count() const
-{
- return SINGLE == _state ? 1 : _ranges.size();
-}
-
-inline bool
-RangeSpec::isValid() const
-{
- return SINGLE == _state || MULTI == _state;
-}
-
-inline RangeSpec::Range&
-RangeSpec::Range::invalidate()
-{
- _min = UINT64_MAX;
- _max = 1;
- return *this;
-}
-
-inline bool
-RangeSpec::Range::isSuffix() const
-{
- return 0 == _max && _min > 0;
-}
-
-inline bool
-RangeSpec::Range::isValid() const
-{
- return _min <= _max || this->isSuffix();
-}
-
-inline bool
-RangeSpec::Range::finalize(uint64_t len)
-{
- ink_assert(len > 0);
- bool zret = true; // is this range satisfiable for @a len?
- if (this->isSuffix()) {
- _max = len - 1;
- _min = _min > len ? 0 : len - _min;
- } else if (_min < len) {
- _max = MIN(_max,len);
- } else {
- this->invalidate();
- zret = false;
- }
- return zret;
-}
#endif // __CACHE_DEFS_H__
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/iocore/cache/P_CacheHttp.h
----------------------------------------------------------------------
diff --git a/iocore/cache/P_CacheHttp.h b/iocore/cache/P_CacheHttp.h
index e64a200..f76e842 100644
--- a/iocore/cache/P_CacheHttp.h
+++ b/iocore/cache/P_CacheHttp.h
@@ -34,7 +34,6 @@ typedef URL CacheURL;
typedef HTTPHdr CacheHTTPHdr;
typedef HTTPInfo CacheHTTPInfo;
-
#define OFFSET_BITS 24
enum
{
@@ -88,6 +87,80 @@ struct CacheHTTPInfoVector
Ptr<RefCountObj> vector_buf;
};
+/** Range operation tracking.
+
+ This holds a range specification. It also tracks the current object offset and the individual range.
+
+ For simplification of the logic that uses this class it will pretend to be a single range of
+ the object size if it is empty. To return the correct response we still need to distinuish
+ those two cases.
+*/
+class CacheRange
+{
+ public:
+ typedef CacheRange self; ///< Self reference type.
+
+ /// Default constructor
+ CacheRange() : _offset(0), _idx(-1), _r(NULL), _ct_field(NULL) { }
+
+ /// Set the internal range spec pointer to @a src.
+ self& setRangeSpec(HTTPRangeSpec* src);
+
+ /// Test if the range should be active (used).
+ /// @internal This means it has ranges and should be used to do seeks on the content.
+ bool isActive() const;
+
+ /// Test for multiple ranges.
+ bool isMulti() const;
+
+ /// Get the current object offset
+ uint64_t getOffset() const;
+
+ /// Get the current range index.
+ int getIdx() const;
+
+ /// Get the remaining contiguous bytes for the current range.
+ uint64_t getRemnantSize() const;
+
+ /** Advance @a size bytes in the range spec.
+
+ @return The resulting offset in the object.
+ */
+ uint64_t consume(uint64_t size);
+
+ /** Apply a @a src range and content @a len to the contained range spec.
+
+ @return @c true if successfully applied, @c false otherwise.
+ */
+ bool apply(HTTPRangeSpec const& src, uint64_t len);
+
+ /** Get the range boundary string.
+ @a len if not @c NULL receives the length of the string.
+ */
+ char const* getBoundaryStr(int* len) const;
+
+ /** Generate the range boundary string */
+ void generateBoundaryStr(CacheKey const& key);
+
+ /** Stash the Content-Type field pointer from a @a header.
+
+ @return @c true if a Content-Type field was found in @a header, @c false if not.
+ */
+ bool setContentType(HTTPHdr* header);
+
+ /** Calculate the effective HTTP content length value.
+ */
+ uint64_t calcContentLength() const;
+
+ protected:
+ uint64_t _len; ///< Total object length.
+ uint64_t _offset; ///< Offset in content.
+ int _idx; ///< Current range index. (< 0 means not in a range)
+ HTTPRangeSpec* _r; ///< The actual ranges.
+ MIMEField* _ct_field; ///< Content-Type field.
+ char _boundary[HTTP_RANGE_BOUNDARY_LEN];
+};
+
TS_INLINE CacheHTTPInfo *
CacheHTTPInfoVector::get(int idx)
{
@@ -96,4 +169,61 @@ CacheHTTPInfoVector::get(int idx)
return &data[idx].alternate;
}
+inline CacheRange&
+CacheRange::setRangeSpec(HTTPRangeSpec* src)
+{
+ _r = src;
+ return *this;
+}
+
+inline bool
+CacheRange::apply(HTTPRangeSpec const& src, uint64_t len)
+{
+ return _r && _r->apply(src, len);
+}
+
+inline bool
+CacheRange::isActive() const
+{
+ return _r && (_r->isSingle() || _r->isMulti());
+}
+
+inline uint64_t
+CacheRange::getOffset() const
+{
+ return _offset;
+}
+
+inline int
+CacheRange::getIdx() const
+{
+ return _idx;
+}
+
+inline uint64_t
+CacheRange::getRemnantSize() const
+{
+ uint64_t zret = 0;
+
+ if (!_r || _r->isEmpty())
+ zret = _len - _offset;
+ else if (_r->isValid() && 0 <= _idx && _idx < static_cast<int>(_r->count()))
+ zret = ((*_r)[_idx]._max - _offset) + 1;
+
+ return zret;
+}
+
+inline char const*
+CacheRange::getBoundaryStr(int* len) const
+{
+ if (len) *len = sizeof(_boundary);
+ return _boundary;
+}
+
+inline bool
+CacheRange::isMulti() const
+{
+ return _r && _r->isMulti();
+}
+
#endif /* __CACHE_HTTP_H__ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/iocore/cache/P_CacheInternal.h
----------------------------------------------------------------------
diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h
index 2de18a3..e386c98 100644
--- a/iocore/cache/P_CacheInternal.h
+++ b/iocore/cache/P_CacheInternal.h
@@ -233,46 +233,6 @@ extern int cache_config_mutex_retry_delay;
extern int good_interim_disks;
#endif
-/** Range operation tracking.
-
- This holds a range specification. It also tracks the current object offset and the individual range.
-
- For simplification of the logic that uses this class it will pretend to be a single range of
- the object size if it is empty. To return the correct response we still need to distinuish
- those two cases.
-*/
-class CacheRange : public RangeSpec
-{
- public:
- typedef CacheRange self; ///< Self reference type.
- typedef RangeSpec super; ///< Parent type.
-
- /// Default constructor
- CacheRange() : super(), _offset(0), _idx(-1) { }
-
- /// Convert to specific values based on content @a length.
- bool finalize(uint64_t length);
-
- /// Get the current object offset
- uint64_t getOffset() const;
-
- /// Get the current range index.
- int getIdx() const;
-
- /// Get the remaining contiguous bytes for the current range.
- uint64_t getRemnantSize() const;
-
- /** Advance @a size bytes in the range spec.
-
- @return The resulting offset in the object.
- */
- uint64_t consume(uint64_t size);
-
- protected:
- uint64_t _len; ///< Total object length.
- uint64_t _offset; ///< Offset in content.
- int _idx; ///< Current range index. (< 0 means not in a range)
-};
// CacheVC
struct CacheVC: public CacheVConnection
@@ -414,7 +374,11 @@ struct CacheVC: public CacheVConnection
/// The table of @a frags and its @a count must be provided.
int frag_idx_for_offset(HTTPInfo::FragOffset* frags, int count, uint64_t offset);
+ virtual char const* get_http_range_boundary_string(int* len) const;
+ virtual uint64_t get_http_content_size();
+
#endif
+
virtual bool is_pread_capable();
virtual bool set_pin_in_cache(time_t time_pin);
virtual time_t get_pin_in_cache();
@@ -503,6 +467,7 @@ struct CacheVC: public CacheVConnection
uint64_t total_len; // total length written and available to write
uint64_t doc_len; // total_length (of the selected alternate for HTTP)
uint64_t update_len;
+ CacheRange resp_range; ///< Tracking information for range data for response.
/// The offset in the content of the first byte beyond the end of the current fragment.
/// @internal This seems very weird but I couldn't figure out how to keep the more sensible
/// lower bound correctly updated.
@@ -516,12 +481,7 @@ struct CacheVC: public CacheVConnection
int header_to_write_len;
void *header_to_write;
short writer_lock_retry;
- /* Range specs for range based operations.
- @a req_rng is the range spec in the request from the User Agent.
- @a rsp_rng is the range spec sent to the origin server.
- */
- CacheRange req_rs;
- CacheRange rsp_rs;
+
#if TS_USE_INTERIM_CACHE == 1
InterimCacheVol *interim_vol;
MigrateToInterimCache *mts;
@@ -688,8 +648,6 @@ free_CacheVC(CacheVC *cont)
cont->alternate_index = CACHE_ALT_INDEX_DEFAULT;
if (cont->scan_vol_map)
ats_free(cont->scan_vol_map);
- cont->req_rs.~CacheRange();
- cont->rsp_rs.~CacheRange();
memset((char *) &cont->vio, 0, cont->size_to_init);
#ifdef CACHE_STAT_PAGES
ink_assert(!cont->stat_link.next && !cont->stat_link.prev);
@@ -1419,29 +1377,6 @@ local_cache()
return theCache;
}
-inline uint64_t
-CacheRange::getOffset() const
-{
- return _offset;
-}
-
-inline int
-CacheRange::getIdx() const
-{
- return _idx;
-}
-
-inline uint64_t
-CacheRange::getRemnantSize() const
-{
- uint64_t zret = 0;
- if (this->isEmpty()) zret = _len - _offset;
- else if (this->isSingle()) zret = (_single._max - _offset) + 1;
- else if (this->isMulti() && 0 <= _idx && _idx < static_cast<int>(_ranges.size()))
- zret = (_ranges[_idx]._max - _offset) + 1;
- return zret;
-}
-
LINK_DEFINITION(CacheVC, opendir_link)
#endif /* _P_CACHE_INTERNAL_H__ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/lib/ts/InkErrno.h
----------------------------------------------------------------------
diff --git a/lib/ts/InkErrno.h b/lib/ts/InkErrno.h
index bb9bfb1..45a846d 100644
--- a/lib/ts/InkErrno.h
+++ b/lib/ts/InkErrno.h
@@ -66,7 +66,7 @@
#define ECACHE_NOT_READY (CACHE_ERRNO+7)
#define ECACHE_ALT_MISS (CACHE_ERRNO+8)
#define ECACHE_BAD_READ_REQUEST (CACHE_ERRNO+9)
-#define ECACHE_BAD_REQUEST_RANGE (CACHE_ERRNO+10)
+#define ECACHE_UNSATISFIABLE_RANGE (CACHE_ERRNO+10)
#define EHTTP_ERROR (HTTP_ERRNO+0)
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/proxy/hdrs/HTTP.cc
----------------------------------------------------------------------
diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc
index ce28ba7..06bebaa 100644
--- a/proxy/hdrs/HTTP.cc
+++ b/proxy/hdrs/HTTP.cc
@@ -2132,3 +2132,177 @@ HTTPInfo::push_frag_offset(FragOffset offset) {
m_alt->m_frag_offsets[m_alt->m_frag_offset_count++] = offset;
}
+
+/***********************************************************************
+ * *
+ * R A N G E S U P P O R T *
+ * *
+ ***********************************************************************/
+
+bool
+HTTPRangeSpec::parse(char const* v, int len)
+{
+ // Maximum # of digits permitted for an offset. Avoid issues with overflow.
+ static size_t const MAX_DIGITS = 15;
+ ts::ConstBuffer src(v, len);
+ size_t n;
+
+ _state = EMPTY;
+ src.skip(&ParseRules::is_ws);
+
+ if (src.size() > sizeof(HTTP_LEN_BYTES)+1 &&
+ 0 == memcmp(src.data(), HTTP_VALUE_BYTES, HTTP_LEN_BYTES) && '=' == src[HTTP_LEN_BYTES]
+ ) {
+ _state = INVALID; // something, it needs to be correct.
+ src += HTTP_LEN_BYTES+1;
+ while (src) {
+ ts::ConstBuffer max = src.splitOn(',');
+
+ if (!max) { // no comma so everything in @a src should be processed as a single range.
+ max = src;
+ src.reset();
+ }
+
+ ts::ConstBuffer min = max.splitOn('-');
+
+ src.skip(&ParseRules::is_ws);
+ // Spec forbids whitspace anywhere in the range element.
+
+ if (min) {
+ if (ParseRules::is_digit(*min) && min.size() <= MAX_DIGITS) {
+ uint64_t low = ats_strto64(min.data(), min.size(), &n);
+ if (n < min.size()) break; // extra cruft in range, not even ws allowed
+ if (max) {
+ if (ParseRules::is_digit(*max) && max.size() <= MAX_DIGITS) {
+ uint64_t high = ats_strto64(max.data(), max.size(), &n);
+ if (n < max.size() && (max += n).skip(&ParseRules::is_ws))
+ break; // non-ws cruft after maximum
+ else
+ this->add(low, high);
+ } else {
+ break; // invalid characters for maximum
+ }
+ } else {
+ this->add(low, UINT64_MAX); // "X-" : "offset X to end of content"
+ }
+ } else {
+ break; // invalid characters for minimum
+ }
+ } else {
+ if (max) {
+ if (ParseRules::is_digit(*max) && max.size() <= MAX_DIGITS) {
+ uint64_t high = ats_strto64(max.data(), max.size(), &n);
+ if (n < max.size() && (max += n).skip(&ParseRules::is_ws)) {
+ break; // cruft after end of maximum
+ } else {
+ this->add(high, 0);
+ }
+ } else {
+ break; // invalid maximum
+ }
+ }
+ }
+ }
+ if (src) _state = INVALID; // didn't parse everything, must have been an error.
+ }
+ return _state != INVALID;
+}
+
+HTTPRangeSpec&
+HTTPRangeSpec::add(uint64_t low, uint64_t high)
+{
+ if (MULTI == _state) {
+ _ranges.push_back(Range(low, high));
+ } else if (SINGLE == _state) {
+ _ranges.push_back(_single);
+ _ranges.push_back(Range(low,high));
+ _state = MULTI;
+ } else {
+ _single._min = low;
+ _single._max = high;
+ _state = SINGLE;
+ }
+ return *this;
+}
+
+bool
+HTTPRangeSpec::apply(self const& that, uint64_t len)
+{
+ _state = that._state;
+ if (INVALID == _state || EMPTY == _state) {
+ // nothing - simplifying later logic.
+ } else if (0 == len) {
+ /* Must special case zero length content
+ - suffix ranges are OK but other ranges are not.
+ - Best option is to return a 200 (not 206 or 416) for all suffix range spec on zero length content.
+ (this is what Apache HTTPD does)
+ - So, mark result as either @c UNSATISFIABLE or @c EMPTY, don't bother copying any ranges.
+ */
+ _state = EMPTY;
+ if (!that._single.isSuffix()) _state = UNSATISFIABLE;
+ for ( RangeBox::const_iterator spot = that._ranges.begin(), limit = that._ranges.end() ; spot != limit && EMPTY == _state ; ++spot ) {
+ if (!spot->isSuffix()) _state = UNSATISFIABLE;
+ }
+ } else if (that.isSingle()) {
+ _single = that._single;
+ if (!_single.apply(len)) _state = UNSATISFIABLE;
+ } else { // gotta be MULTI
+ _ranges.reserve(that._ranges.size());
+ for ( RangeBox::const_iterator spot = that._ranges.begin(), limit = that._ranges.end() ; spot != limit ; ++spot ) {
+ Range r(*spot);
+ if (r.apply(len)) _ranges.push_back(r);
+ }
+ if (_ranges.size() > 0) {
+ _single = _ranges[0];
+ if (_ranges.size() == 1) _state = SINGLE;
+ } else {
+ _state = UNSATISFIABLE;
+ }
+ }
+ return this->isValid();
+}
+
+HTTPRangeSpec&
+HTTPHdr::getRangeSpec()
+{
+ if (!m_range_parsed && HTTP_TYPE_REQUEST == m_http->m_polarity) {
+ MIMEField* f = this->field_find(MIME_FIELD_RANGE, MIME_LEN_RANGE);
+ if (f) {
+ int len;
+ char const* value = f->value_get(&len);
+ if (value) {
+ m_range_spec.parse(value,len);
+ }
+ }
+ }
+ m_range_parsed = true;
+ return m_range_spec;
+}
+
+int
+Calc_Digital_Length(uint64_t x)
+{
+ char buff[32]; // big enough for 64 bit #
+ return snprintf(buff, sizeof(buff), "%" PRIu64, x);
+}
+
+uint64_t
+HTTPRangeSpec::calcContentLength(uint64_t object_size, uint64_t ct_len) const
+{
+ uint64_t size = object_size;
+ size_t nr = this->count();
+
+ if (nr >= 1) {
+ size = this->size();
+ if (nr > 1) {
+ size_t l_size = Calc_Digital_Length(object_size);
+ // CR LF "--" boundary-string CR LF "Content-Range" ": " "bytes " X "-" Y "/" Z
+ uint64_t sep_size = 2 + 2 + HTTP_RANGE_BOUNDARY_LEN + 2 + MIME_LEN_CONTENT_RANGE + 2 + HTTP_LEN_BYTES + 1 + l_size + 1 +l_size + 1 + l_size + 2;
+
+ if (ctf) sep_size += MIME_LEN_CONTENT_TYPE + 2 + ct_len + 2;
+ size += nr * sep_size;
+ }
+ }
+ return size;
+}
+
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/proxy/hdrs/HTTP.h
----------------------------------------------------------------------
diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h
index f9175e5..cbe0316 100644
--- a/proxy/hdrs/HTTP.h
+++ b/proxy/hdrs/HTTP.h
@@ -25,6 +25,7 @@
#define __HTTP_H__
#include <assert.h>
+#include <vector>
#include "Arena.h"
#include "INK_MD5.h"
#include "MIME.h"
@@ -429,6 +430,8 @@ extern int HTTP_LEN_S_MAXAGE;
extern int HTTP_LEN_NEED_REVALIDATE_ONCE;
extern int HTTP_LEN_100_CONTINUE;
+static size_t const HTTP_RANGE_BOUNDARY_LEN = 32 + 2 + 16;
+
/* Private */
void http_hdr_adjust(HTTPHdrImpl *hdrp, int32_t offset, int32_t length, int32_t delta);
@@ -510,6 +513,138 @@ public:
int32_t m_version;
};
+/** A set of content ranges.
+
+ This represents the data for an HTTP range specification.
+ On a request this contains the request ranges. On a response it is the actual ranges in the
+ response, which are the requested ranges modified by the actual content length.
+*/
+struct HTTPRangeSpec {
+ typedef HTTPRangeSpec self;
+
+ /** A range of bytes in an object.
+
+ If @a _min > 0 and @a _max == 0 the range is backwards and counts from the
+ end of the object. That is (100,0) means the last 100 bytes of content.
+ */
+ struct Range {
+ uint64_t _min;
+ uint64_t _max;
+
+ /// Default constructor - invalid range.
+ Range() : _min(UINT64_MAX), _max(1) { }
+ /// Construct as the range ( @a low .. @a high )
+ Range(uint64_t low, uint64_t high) : _min(low), _max(high) {}
+
+ /// Test if this range is a suffix range.
+ bool isSuffix() const;
+ /// Test if this range is a valid range.
+ bool isValid() const;
+ /// Get the size (in bytes) of the range.
+ uint64_t size() const;
+ /** Convert range to absolute values for a content length of @a len.
+
+ @return @c true if the range was valid for @a len, @c false otherwise.
+ */
+ bool apply(uint64_t len);
+
+ /// Force the range to an empty state.
+ Range& invalidate();
+ };
+
+ /// Range iteration type.
+ typedef Range* iterator;
+
+ /// Current state of the overall specification.
+ /// @internal We can distinguish between @c SINGLE and @c MULTI by looking at the
+ /// size of @a _ranges but we need this to mark @c EMPTY vs. not.
+ enum State {
+ EMPTY, ///< No range.
+ INVALID, ///< Range parsing failed.
+ UNSATISFIABLE, ///< Content length application failed.
+ SINGLE, ///< Single range.
+ MULTI, ///< Multiple ranges.
+ } _state;
+
+ /// The first range value.
+ /// By separating this out we can avoid allocation in the case of a single
+ /// range value, which is by far the most common ( > 99% in my experience).
+ Range _single;
+ /// Storage for range values.
+ typedef std::vector<Range> RangeBox;
+ /// The first range is copied here if there is more than one (to simplify).
+ RangeBox _ranges;
+
+ /// Default constructor - invalid range
+ HTTPRangeSpec();
+
+ /** Parse a range field @a value and update @a this with the results.
+ @return @c true if @a value was a valid range specifier, @c false otherwise.
+ */
+ bool parse(char const* value, int len);
+
+ /** Copy ranges from @a while applying them to the content @a length.
+
+ Ranges are copied if valid for @a length and converted to absolute offsets. The number of ranges
+ after application may be less than the @a src number of ranges. In addition ranges will be clipped
+ to @a length.
+
+ @return @c true if the range spec is satisfiable, @c false otherwise.
+ Note a range spec with no ranges is always satisfiable and that suffix ranges are also
+ always satisfiable.
+ */
+ bool apply(self const& that, uint64_t length);
+
+ /** Number of distinct ranges.
+ @return Number of ranges.
+ */
+ size_t count() const;
+
+ /// Get the size (in bytes) of the ranges.
+ uint64_t size() const;
+
+ /// If this is a valid single range specification.
+ bool isSingle() const;
+
+ /// If this is a valid multi range specification.
+ bool isMulti() const;
+
+ /// Test if this contains at least one valid range.
+ bool hasRanges() const;
+
+ /// Test if this is a well formed range (may be empty).
+ bool isValid() const;
+
+ /// Test if this is a valid but empty range spec.
+ bool isEmpty() const;
+
+ /// Test if this is an unsatisfied range.
+ bool isUnsatisfied() const;
+
+ /// Access the range at index @a idx.
+ Range& operator [] (int n);
+
+ /** Calculate the content length for this range specification.
+
+ @note If a specific content length has not been @c apply 'd this will not produce
+ a usable result.
+
+ @return The content length for the ranges including the range separators.
+ */
+ uint64_t calcContentLength(
+ uint64_t base_content_size, ///< Content size w/o ranges.
+ uint64_t ct_len ///< Length of Content-Type field value.
+ ) const;
+
+ /// Iterator for first range.
+ iterator begin();
+ /// Iterator past last range.
+ iterator end();
+
+protected:
+ self& add(uint64_t low, uint64_t high);
+};
+
class IOBufferReader;
class HTTPHdr: public MIMEHdr
@@ -529,6 +664,15 @@ public:
/// also had a port, @c false otherwise.
mutable bool m_port_in_header;
+ /// Parsed data from the RANGE field.
+ /// For requests, this is the RANGE field specification.
+ /// For responses, this is the RANGE field applied to the content length.
+ HTTPRangeSpec m_range_spec;
+ /// This is the content type, detached from the response header, if needed.
+ MIMEField* m_range_content_type;
+ /// Have we parsed the range field yet?
+ bool m_range_parsed;
+
HTTPHdr();
~HTTPHdr();
@@ -651,6 +795,12 @@ public:
const char *reason_get(int *length);
void reason_set(const char *value, int length);
+ /// Get the internal @c HTTPRangeSpec instance.
+ HTTPRangeSpec& getRangeSpec();
+ /// Locate and parse (if present) the @c Range header field.
+ /// The results are put in to the internal @c HTTPRangeSpec instance.
+ bool parse_range();
+
MIMEParseResult parse_req(HTTPParser *parser, const char **start, const char *end, bool eof);
MIMEParseResult parse_resp(HTTPParser *parser, const char **start, const char *end, bool eof);
@@ -799,7 +949,7 @@ HTTPVersion::operator <=(const HTTPVersion & hv) const
inline
HTTPHdr::HTTPHdr()
- : MIMEHdr(), m_http(NULL), m_url_cached(), m_target_cached(false)
+ : MIMEHdr(), m_http(NULL), m_url_cached(), m_target_cached(false), m_range_content_type(NULL), m_range_parsed(false)
{ }
@@ -1523,4 +1673,132 @@ HTTPInfo::get_frag_offset_count() {
return m_alt ? m_alt->m_frag_offset_count : 0;
}
+inline
+HTTPRangeSpec::HTTPRangeSpec() : _state(EMPTY)
+{
+}
+
+inline bool
+HTTPRangeSpec::isSingle() const
+{
+ return SINGLE == _state;
+}
+
+inline bool
+HTTPRangeSpec::isMulti() const
+{
+ return MULTI == _state;
+}
+
+inline bool
+HTTPRangeSpec::isEmpty() const
+{
+ return EMPTY == _state;
+}
+
+inline bool
+HTTPRangeSpec::isUnsatisfied() const
+{
+ return UNSATISFIABLE == _state;
+}
+
+inline size_t
+HTTPRangeSpec::count() const
+{
+ return SINGLE == _state ? 1 : _ranges.size();
+}
+
+inline bool
+HTTPRangeSpec::hasRanges() const
+{
+ return SINGLE == _state || MULTI == _state;
+}
+
+inline bool
+HTTPRangeSpec::isValid() const
+{
+ return SINGLE == _state || MULTI == _state || EMPTY == _state;
+}
+
+inline HTTPRangeSpec::Range&
+HTTPRangeSpec::Range::invalidate()
+{
+ _min = UINT64_MAX;
+ _max = 1;
+ return *this;
+}
+
+inline bool
+HTTPRangeSpec::Range::isSuffix() const
+{
+ return 0 == _max && _min > 0;
+}
+
+inline bool
+HTTPRangeSpec::Range::isValid() const
+{
+ return _min <= _max || this->isSuffix();
+}
+
+inline uint64_t
+HTTPRangeSpec::Range::size() const
+{
+ return 1 + (_max - _min);
+}
+
+inline uint64_t
+HTTPRangeSpec::size() const
+{
+ uint64_t size = 0;
+ if (this->isSingle()) size = _single.size();
+ else if (this->isMulti()) {
+ for ( RangeBox::const_iterator spot = _ranges.begin(), limit = _ranges.end() ; spot != limit ; ++spot)
+ size += spot->size();
+ }
+ return size;
+}
+
+inline bool
+HTTPRangeSpec::Range::apply(uint64_t len)
+{
+ ink_assert(len > 0);
+ bool zret = true; // is this range satisfiable for @a len?
+ if (this->isSuffix()) {
+ _max = len - 1;
+ _min = _min > len ? 0 : len - _min;
+ } else if (_min < len) {
+ _max = MIN(_max,len);
+ } else {
+ this->invalidate();
+ zret = false;
+ }
+ return zret;
+}
+
+inline HTTPRangeSpec::Range&
+HTTPRangeSpec::operator [] (int n)
+{
+ return SINGLE == _state ? _single : _ranges[n];
+}
+
+inline HTTPRangeSpec::iterator
+HTTPRangeSpec::begin()
+{
+ switch (_state) {
+ case SINGLE: return &_single;
+ case MULTI: return &(*(_ranges.begin()));
+ default: return NULL;
+ }
+}
+
+inline HTTPRangeSpec::iterator
+HTTPRangeSpec::end()
+{
+ switch (_state) {
+ case SINGLE: return (&_single)+1;
+ case MULTI: return &(*(_ranges.end()));
+ default: return NULL;
+ }
+}
+
#endif /* __HTTP_H__ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/7b1f37d9/proxy/http/HttpTransact.cc
----------------------------------------------------------------------
diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc
index c348c29a..b1f51c3 100644
--- a/proxy/http/HttpTransact.cc
+++ b/proxy/http/HttpTransact.cc
@@ -47,8 +47,7 @@
#include "HttpClientSession.h"
#include "I_Machine.h"
-static char range_type[] = "multipart/byteranges; boundary=RANGE_SEPARATOR";
-#define RANGE_NUMBERS_LENGTH 60
+static char const HTTP_RANGE_MULTIPART_CONTENT_TYPE[] = "multipart/byteranges; boundary=";
#define HTTP_INCREMENT_TRANS_STAT(X) update_stat(s, X, 1);
#define HTTP_SUM_TRANS_STAT(X,S) update_stat(s, X, (ink_statval_t) S);
@@ -6581,14 +6580,13 @@ HttpTransact::handle_content_length_header(State* s, HTTPHdr* header, HTTPHdr* b
case SOURCE_HTTP_ORIGIN_SERVER:
// We made our decision about whether to trust the
// response content length in init_state_vars_from_response()
- if (s->range_setup != HttpTransact::RANGE_NOT_TRANSFORM_REQUESTED)
- break;
+ break;
case SOURCE_CACHE:
// if we are doing a single Range: request, calculate the new
// C-L: header
- if (s->range_setup == HttpTransact::RANGE_NOT_TRANSFORM_REQUESTED) {
- change_response_header_because_of_range_request(s,header);
+ if (base->hasRanges())
+ change_response_header_because_of_range_request(s,header,base);
s->hdr_info.trust_response_cl = true;
}
////////////////////////////////////////////////
@@ -8760,8 +8758,10 @@ HttpTransact::delete_warning_value(HTTPHdr* to_warn, HTTPWarningCode warning_cod
void
HttpTransact::change_response_header_because_of_range_request(State *s, HTTPHdr * header)
{
- MIMEField *field;
+ MIMEField *field = header->field_find(MILE_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
char *reason_phrase;
+ HTTPHdr* cached_response = cache_sm.cache_read_vc->get_response();
+ HTTPRangeSpec& rs = cached_response->getRangeSpec();
Debug("http_trans", "Partial content requested, re-calculating content-length");
@@ -8770,35 +8770,31 @@ HttpTransact::change_response_header_because_of_range_request(State *s, HTTPHdr
header->reason_set(reason_phrase, strlen(reason_phrase));
// set the right Content-Type for multiple entry Range
- if (s->num_range_fields > 1) {
- field = header->field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
+ if (rs.isMulti()) { // means we need a boundary string.
+ char buff[(sizeof(HTTP_MULTIPART_RANGE_CONTENT_TYPE)-1) + HTTP_RANGE_BOUNDARY_LEN];
if (field != NULL)
header->field_delete(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
field = header->field_create(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
- field->value_append(header->m_heap, header->m_mime, range_type, sizeof(range_type) - 1);
+ snprintf(buff, sizeof(buff), "%s%.*s", HTTP_MULTIPART_RANGE_CONTENT_TYPE, HTTP_RANGE_BOUNDARY_LEN, cache_sm.cache_read_vc->getBoundaryStr());
+ field->value_append(header->m_heap, header->m_mime, buff, sizeof(buff));
header->field_attach(field);
- // TODO: There's a known bug here where the Content-Length is not correct for multi-part
- // Range: requests.
- header->set_content_length(s->range_output_cl);
- } else {
- if (s->cache_info.object_read && s->cache_info.object_read->valid()) {
- // TODO: It's unclear under which conditions we need to update the Content-Range: header,
- // many times it's already set correctly before calling this. For now, always try do it
- // when we have the information for it available.
- // TODO: Also, it's unclear as to why object_read->valid() is not always true here.
- char numbers[RANGE_NUMBERS_LENGTH];
- header->field_delete(MIME_FIELD_CONTENT_RANGE, MIME_LEN_CONTENT_RANGE);
- field = header->field_create(MIME_FIELD_CONTENT_RANGE, MIME_LEN_CONTENT_RANGE);
- snprintf(numbers, sizeof(numbers), "bytes %" PRId64"-%" PRId64"/%" PRId64, s->ranges[0]._start, s->ranges[0]._end,
- s->cache_info.object_read->object_size_get());
- field->value_set(header->m_heap, header->m_mime, numbers, strlen(numbers));
- header->field_attach(field);
- }
- // Always update the Content-Length: header.
- header->set_content_length(s->range_output_cl);
+ header->set_content_length(cached_response()->calcContentLength());
+ } else if (rs.isSingle()) {
+ int n;
+ char numbers[RANGE_NUMBERS_LENGTH];
+ header->field_delete(MIME_FIELD_CONTENT_RANGE, MIME_LEN_CONTENT_RANGE);
+ field = header->field_create(MIME_FIELD_CONTENT_RANGE, MIME_LEN_CONTENT_RANGE);
+ n = snprintf( numbers, sizeof(numbers), "%s %" PRIu64"-%" PRIu64"/%" PRId64
+ , HTTP_VALUE_BYTES
+ , (*rs)[0]._min, (*rs)[0]._max
+ , cached_response->get_content_length()
+ );
+ field->value_set(header->m_heap, header->m_mime, numbers, n);
+ header->field_attach(field);
+ header->set_content_length(cached_response()->calcContentLength());
}
}
[5/8] trafficserver git commit: Checkpoint.
Posted by am...@apache.org.
Checkpoint.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/45dc0c60
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/45dc0c60
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/45dc0c60
Branch: refs/heads/ts-974-multi-range-read
Commit: 45dc0c602e13a4dcf3a0681c195f3fd5f828a152
Parents: 1e43f79
Author: Alan M. Carroll <so...@yahoo-inc.com>
Authored: Thu Nov 20 15:43:58 2014 -0600
Committer: Alan M. Carroll <so...@yahoo-inc.com>
Committed: Sat Dec 6 11:56:03 2014 -0600
----------------------------------------------------------------------
iocore/cache/CacheHttp.cc | 29 +++++++++++++++++++++++++++++
iocore/cache/I_CacheDefs.h | 41 ++++++++++++++++++++++++++++++++++++++++-
lib/ts/TsBuffer.h | 38 +++++++++++++++++++++++++++++++++++---
3 files changed, 104 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/45dc0c60/iocore/cache/CacheHttp.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheHttp.cc b/iocore/cache/CacheHttp.cc
index 6bcc4fc..31531f2 100644
--- a/iocore/cache/CacheHttp.cc
+++ b/iocore/cache/CacheHttp.cc
@@ -256,6 +256,33 @@ CacheHTTPInfoVector::get_handles(const char *buf, int length, RefCountObj * bloc
return ((caddr_t) buf - (caddr_t) start);
}
+/*-------------------------------------------------------------------------
+ -------------------------------------------------------------------------*/
+bool
+RangeSpec::parse(char const* v, int len)
+{
+ char const PREFIX[] = { 'b', 'y', 't', 'e', 's', '=' };
+ ts::ConstBuffer src(v, len);
+
+ src.skip(&ParseRules::is_ws);
+
+ if (src.size() > sizeof(PREFIX) && 0 == memcmp(src.data(), PREFIX, sizeof(PREFIX))) {
+ src += sizeof(PREFIX);
+ while (src) {
+ ts::ConstBuffer max = src.splitOn(',');
+ ts::ConstBuffer min = max.splitOn('-');
+ src.skip(&ParseRules::is_ws);
+
+ if (min) {
+ } else {
+ }
+ }
+ }
+ return true;
+}
+/*-------------------------------------------------------------------------
+ -------------------------------------------------------------------------*/
+
#else //HTTP_CACHE
CacheHTTPInfoVector::CacheHTTPInfoVector()
@@ -351,5 +378,7 @@ CacheHTTPInfoVector::get_handles(const char */* buf ATS_UNUSED */, int /* length
ink_assert(0);
return 0;
}
+/*-------------------------------------------------------------------------
+ -------------------------------------------------------------------------*/
#endif //HTTP_CACHE
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/45dc0c60/iocore/cache/I_CacheDefs.h
----------------------------------------------------------------------
diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h
index bd5bf46..84887de 100644
--- a/iocore/cache/I_CacheDefs.h
+++ b/iocore/cache/I_CacheDefs.h
@@ -21,7 +21,7 @@
limitations under the License.
*/
-
+#include <vector>
#ifndef _I_CACHE_DEFS_H__
#define _I_CACHE_DEFS_H__
@@ -131,4 +131,43 @@ typedef CryptoHash CacheKey;
word(2) - tag (lower bits), hosttable hash (upper bits)
word(3) - ram cache hash, lookaside cache
*/
+
+
+/** A range specification.
+
+ This represents the data for an HTTP range specification.
+*/
+struct RangeSpec {
+ /** A range of bytes in an object.
+
+ If @a _min > @a _max this means the range is backwards and counts from the
+ end of the object. That is (1,0) means the last byte of content.
+ */
+ struct Descriptor {
+ uint64_t _min;
+ uint64_t _max;
+ };
+
+ enum State {
+ EMPTY, ///< No range.
+ SINGLE, ///< Single range.
+ MULTI, ///< Multiple ranges.
+ } _state;
+
+ /// The first range value.
+ /// By separating this out we can avoid allocation in the case of a single
+ /// range value, which is by far the most common ( > 99% in my experience).
+ Descriptor _singleton;
+ /// Storage for range values past the first one.
+ std::vector<Descriptor> _ranges;
+
+ RangeSpec() : _state(EMPTY)
+ {}
+
+ /** Parse a range field and update @a this with the results.
+ @return @c true if @a v was a valid range specifier, @c false otherwise.
+ */
+ bool parse(char const* v, int len);
+};
+
#endif // __CACHE_DEFS_H__
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/45dc0c60/lib/ts/TsBuffer.h
----------------------------------------------------------------------
diff --git a/lib/ts/TsBuffer.h b/lib/ts/TsBuffer.h
index d4fac87..840c7bb 100644
--- a/lib/ts/TsBuffer.h
+++ b/lib/ts/TsBuffer.h
@@ -224,9 +224,10 @@ namespace ts {
@note This presumes a half open range (start, end]
*/
self& set(
- char const* start, ///< First valid character.
- char const* end ///< First invalid character.
- );
+ char const* start, ///< First valid character.
+ char const* end ///< First invalid character.
+ );
+
/// Reset to empty.
self& reset();
@@ -271,17 +272,20 @@ namespace ts {
@return A buffer containing data up to but not including @a p.
*/
self splitOn(char c);
+
/** Get a trailing segment of the buffer.
@return A buffer that contains all data after @a p.
*/
self after(char const* p) const;
+
/** Get a trailing segment of the buffer.
@return A buffer that contains all data after the first
occurrence of @a c.
*/
self after(char c) const;
+
/** Remove trailing segment.
Data at @a p and beyond is removed from the buffer.
@@ -290,6 +294,23 @@ namespace ts {
@return @a this.
*/
self& clip(char const* p);
+
+ /** Remove initial instances of @a c.
+
+ @return @c true if not all characters were skipped, @c false if all characters matched @a c.
+ */
+ bool skip(char c);
+
+ /** Remove leading characters that satisfy a @a predicate.
+ @return @c true if not all characters were skipped, @c false if all characters matched the @a predicate.
+
+ @internal We template this because the @c ParseRules predicates (which are the usual suspects)
+ return an integral type that is not @c bool.
+ */
+ template <
+ typename BOOL_EQUIV ///< Type that can be automatically converted to bool
+ >
+ bool skip(BOOL_EQUIV (*predicate)(char));
};
// ----------------------------------------------------------
@@ -392,6 +413,17 @@ namespace ts {
}
return *this;
}
+ template < typename BOOL_EQUIV >
+ inline bool ConstBuffer::skip(BOOL_EQUIV (*predicate)(char))
+ {
+ while (*this && predicate(**this)) ++*this;
+ return *this;
+ }
+ inline bool ConstBuffer::skip(char c)
+ {
+ while (*this && c == **this) ++*this;
+ return *this;
+ }
} // end namespace
[7/8] trafficserver git commit: Checkpoint before removing stuff from
HTTPHdr.
Posted by am...@apache.org.
Checkpoint before removing stuff from HTTPHdr.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/67261f10
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/67261f10
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/67261f10
Branch: refs/heads/ts-974-multi-range-read
Commit: 67261f10bd6c5984f16f7fe2b3d60cf99314845e
Parents: 7b1f37d
Author: Alan M. Carroll <so...@yahoo-inc.com>
Authored: Fri Dec 5 07:27:07 2014 -0600
Committer: Alan M. Carroll <so...@yahoo-inc.com>
Committed: Sat Dec 6 11:56:04 2014 -0600
----------------------------------------------------------------------
iocore/cluster/P_ClusterCache.h | 7 +++++++
proxy/hdrs/HTTP.cc | 4 ++--
proxy/hdrs/HTTP.h | 15 +++++++++++----
proxy/http/HttpTransact.cc | 26 ++++++++++++++------------
4 files changed, 34 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/67261f10/iocore/cluster/P_ClusterCache.h
----------------------------------------------------------------------
diff --git a/iocore/cluster/P_ClusterCache.h b/iocore/cluster/P_ClusterCache.h
index 5d6b0b9..c7a3033 100644
--- a/iocore/cluster/P_ClusterCache.h
+++ b/iocore/cluster/P_ClusterCache.h
@@ -352,6 +352,13 @@ struct ClusterVConnectionBase: public CacheVConnection
virtual void do_io_close(int lerrno = -1);
virtual VIO* do_io_pread(Continuation*, int64_t, MIOBuffer*, int64_t);
+ // TODO - fix these to work for cluster.
+ // I think the best approach is to foist the work off to the source peer and have it do
+ // the range formatting which we then just pass through. For now, this just prevents
+ // link problems so I can get the base case to work.
+ virtual char const* get_http_range_boundary_string(int*) const { return NULL; }
+ virtual uint64_t get_http_content_size() { return this->get_object_size(); }
+
// Set the timeouts associated with this connection.
// active_timeout is for the total elasped time of the connection.
// inactivity_timeout is the elapsed time *while an operation was
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/67261f10/proxy/hdrs/HTTP.cc
----------------------------------------------------------------------
diff --git a/proxy/hdrs/HTTP.cc b/proxy/hdrs/HTTP.cc
index 06bebaa..d2d5f69 100644
--- a/proxy/hdrs/HTTP.cc
+++ b/proxy/hdrs/HTTP.cc
@@ -2287,7 +2287,7 @@ Calc_Digital_Length(uint64_t x)
}
uint64_t
-HTTPRangeSpec::calcContentLength(uint64_t object_size, uint64_t ct_len) const
+HTTPRangeSpec::calcContentLength(uint64_t object_size, uint64_t ct_val_len) const
{
uint64_t size = object_size;
size_t nr = this->count();
@@ -2299,7 +2299,7 @@ HTTPRangeSpec::calcContentLength(uint64_t object_size, uint64_t ct_len) const
// CR LF "--" boundary-string CR LF "Content-Range" ": " "bytes " X "-" Y "/" Z
uint64_t sep_size = 2 + 2 + HTTP_RANGE_BOUNDARY_LEN + 2 + MIME_LEN_CONTENT_RANGE + 2 + HTTP_LEN_BYTES + 1 + l_size + 1 +l_size + 1 + l_size + 2;
- if (ctf) sep_size += MIME_LEN_CONTENT_TYPE + 2 + ct_len + 2;
+ sep_size += MIME_LEN_CONTENT_TYPE + 2 + ct_val_len + 2;
size += nr * sep_size;
}
}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/67261f10/proxy/hdrs/HTTP.h
----------------------------------------------------------------------
diff --git a/proxy/hdrs/HTTP.h b/proxy/hdrs/HTTP.h
index cbe0316..559a3b2 100644
--- a/proxy/hdrs/HTTP.h
+++ b/proxy/hdrs/HTTP.h
@@ -633,7 +633,7 @@ struct HTTPRangeSpec {
*/
uint64_t calcContentLength(
uint64_t base_content_size, ///< Content size w/o ranges.
- uint64_t ct_len ///< Length of Content-Type field value.
+ uint64_t ct_val_len ///< Length of Content-Type field value.
) const;
/// Iterator for first range.
@@ -668,8 +668,6 @@ public:
/// For requests, this is the RANGE field specification.
/// For responses, this is the RANGE field applied to the content length.
HTTPRangeSpec m_range_spec;
- /// This is the content type, detached from the response header, if needed.
- MIMEField* m_range_content_type;
/// Have we parsed the range field yet?
bool m_range_parsed;
@@ -797,6 +795,9 @@ public:
/// Get the internal @c HTTPRangeSpec instance.
HTTPRangeSpec& getRangeSpec();
+ /// Check if this response is a partial content response.
+ /// @return @c true if there is a least one range, @c false if not.
+ bool isPartialContent() const;
/// Locate and parse (if present) the @c Range header field.
/// The results are put in to the internal @c HTTPRangeSpec instance.
bool parse_range();
@@ -949,7 +950,7 @@ HTTPVersion::operator <=(const HTTPVersion & hv) const
inline
HTTPHdr::HTTPHdr()
- : MIMEHdr(), m_http(NULL), m_url_cached(), m_target_cached(false), m_range_content_type(NULL), m_range_parsed(false)
+ : MIMEHdr(), m_http(NULL), m_url_cached(), m_target_cached(false), m_range_parsed(false)
{ }
@@ -1801,4 +1802,10 @@ HTTPRangeSpec::end()
}
}
+inline bool
+HTTPHdr::isPartialContent() const
+{
+ return m_range_spec.hasRanges();
+}
+
#endif /* __HTTP_H__ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/67261f10/proxy/http/HttpTransact.cc
----------------------------------------------------------------------
diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc
index b1f51c3..169e9ce 100644
--- a/proxy/http/HttpTransact.cc
+++ b/proxy/http/HttpTransact.cc
@@ -6585,8 +6585,8 @@ HttpTransact::handle_content_length_header(State* s, HTTPHdr* header, HTTPHdr* b
case SOURCE_CACHE:
// if we are doing a single Range: request, calculate the new
// C-L: header
- if (base->hasRanges())
- change_response_header_because_of_range_request(s,header,base);
+ if (base->isPartialContent()) {
+ change_response_header_because_of_range_request(s,header);
s->hdr_info.trust_response_cl = true;
}
////////////////////////////////////////////////
@@ -8758,9 +8758,10 @@ HttpTransact::delete_warning_value(HTTPHdr* to_warn, HTTPWarningCode warning_cod
void
HttpTransact::change_response_header_because_of_range_request(State *s, HTTPHdr * header)
{
- MIMEField *field = header->field_find(MILE_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
+ MIMEField *field = header->field_find(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
char *reason_phrase;
- HTTPHdr* cached_response = cache_sm.cache_read_vc->get_response();
+ CacheVConnection* cache_read_vc = s->state_machine->get_cache_sm().cache_read_vc;
+ HTTPHdr* cached_response = find_appropriate_cached_resp(s);
HTTPRangeSpec& rs = cached_response->getRangeSpec();
Debug("http_trans", "Partial content requested, re-calculating content-length");
@@ -8771,31 +8772,32 @@ HttpTransact::change_response_header_because_of_range_request(State *s, HTTPHdr
// set the right Content-Type for multiple entry Range
if (rs.isMulti()) { // means we need a boundary string.
- char buff[(sizeof(HTTP_MULTIPART_RANGE_CONTENT_TYPE)-1) + HTTP_RANGE_BOUNDARY_LEN];
+ int rbs_len;
+ char const* rbs = cache_read_vc->get_http_range_boundary_string(&rbs_len);
+ char buff[(sizeof(HTTP_RANGE_MULTIPART_CONTENT_TYPE)-1) + HTTP_RANGE_BOUNDARY_LEN];
if (field != NULL)
header->field_delete(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
field = header->field_create(MIME_FIELD_CONTENT_TYPE, MIME_LEN_CONTENT_TYPE);
- snprintf(buff, sizeof(buff), "%s%.*s", HTTP_MULTIPART_RANGE_CONTENT_TYPE, HTTP_RANGE_BOUNDARY_LEN, cache_sm.cache_read_vc->getBoundaryStr());
+ snprintf(buff, sizeof(buff), "%s%.*s", HTTP_RANGE_MULTIPART_CONTENT_TYPE, rbs_len, rbs);
field->value_append(header->m_heap, header->m_mime, buff, sizeof(buff));
header->field_attach(field);
- header->set_content_length(cached_response()->calcContentLength());
} else if (rs.isSingle()) {
int n;
- char numbers[RANGE_NUMBERS_LENGTH];
+ char buff[HTTP_LEN_BYTES + (18+1)*3];
header->field_delete(MIME_FIELD_CONTENT_RANGE, MIME_LEN_CONTENT_RANGE);
field = header->field_create(MIME_FIELD_CONTENT_RANGE, MIME_LEN_CONTENT_RANGE);
- n = snprintf( numbers, sizeof(numbers), "%s %" PRIu64"-%" PRIu64"/%" PRId64
+ n = snprintf(buff, sizeof(buff), "%s %" PRIu64"-%" PRIu64"/%" PRId64
, HTTP_VALUE_BYTES
- , (*rs)[0]._min, (*rs)[0]._max
+ , rs[0]._min, rs[0]._max
, cached_response->get_content_length()
);
- field->value_set(header->m_heap, header->m_mime, numbers, n);
+ field->value_set(header->m_heap, header->m_mime, buff, n);
header->field_attach(field);
- header->set_content_length(cached_response()->calcContentLength());
}
+ header->set_content_length(cache_read_vc->get_http_content_size());
}
#if TS_HAS_TESTS
[4/8] trafficserver git commit: First correct range processing
(single range).
Posted by am...@apache.org.
First correct range processing (single range).
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/d33b6255
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/d33b6255
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/d33b6255
Branch: refs/heads/ts-974-multi-range-read
Commit: d33b6255f031801ed176b3fb77caea09b6640b37
Parents: ee70849
Author: Alan M. Carroll <so...@yahoo-inc.com>
Authored: Mon Dec 1 11:04:56 2014 -0600
Committer: Alan M. Carroll <so...@yahoo-inc.com>
Committed: Sat Dec 6 11:56:03 2014 -0600
----------------------------------------------------------------------
iocore/cache/Cache.cc | 5 ++
iocore/cache/CacheHttp.cc | 40 ++++++++++
iocore/cache/CacheRead.cc | 152 ++++++++++++++++++++----------------
iocore/cache/I_CacheDefs.h | 29 ++++++-
iocore/cache/P_CacheInternal.h | 68 ++++++++++++++--
5 files changed, 220 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d33b6255/iocore/cache/Cache.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc
index 9aece93..de20dc9 100644
--- a/iocore/cache/Cache.cc
+++ b/iocore/cache/Cache.cc
@@ -2751,6 +2751,11 @@ LinterimRead:
io.action = this;
io.thread = mutex->thread_holding->tt == DEDICATED ? AIO_CALLBACK_THREAD_ANY : mutex->thread_holding;
SET_HANDLER(&CacheVC::handleReadDone);
+ {
+ char xt[33];
+ Debug("amc", "cache read : key = %s %" PRId64 " bytes at stripe offset =% " PRId64
+ , key.toHexStr(xt), io.aiocb.aio_nbytes, io.aiocb.aio_offset);
+ }
ink_assert(ink_aio_read(&io) >= 0);
CACHE_DEBUG_INCREMENT_DYN_STAT(cache_pread_count_stat);
return EVENT_CONT;
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d33b6255/iocore/cache/CacheHttp.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheHttp.cc b/iocore/cache/CacheHttp.cc
index 024f5ca..3b86a9b 100644
--- a/iocore/cache/CacheHttp.cc
+++ b/iocore/cache/CacheHttp.cc
@@ -374,6 +374,46 @@ RangeSpec::finalize(uint64_t len)
/*-------------------------------------------------------------------------
-------------------------------------------------------------------------*/
+bool
+CacheRange::finalize(uint64_t len)
+{
+ bool zret = super::finalize(len);
+ if (zret) {
+ if (this->isEmpty()) { // pretend it's one range [0..len)
+ _offset = 0;
+ } else {
+ _idx = 0;
+ _offset = _single._min;
+ }
+ _len = len;
+ }
+ return zret;
+}
+
+uint64_t
+CacheRange::consume(uint64_t size)
+{
+ switch (_state) {
+ case EMPTY: _offset += size; break;
+ case SINGLE: _offset += std::min(size, (_single._max - _offset) + 1 ); break;
+ case MULTI:
+ while (size && _idx < static_cast<int>(_ranges.size())) {
+ uint64_t r = std::min(size, (_ranges[_idx]._max - _offset) + 1);
+ _offset += r;
+ size -= r;
+ if (_offset > _ranges[_idx]._max)
+ _offset = _ranges[++_idx]._min;
+ }
+ break;
+ default: break;
+ }
+
+ return _offset;
+}
+
+/*-------------------------------------------------------------------------
+ -------------------------------------------------------------------------*/
+
#else //HTTP_CACHE
CacheHTTPInfoVector::CacheHTTPInfoVector()
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d33b6255/iocore/cache/CacheRead.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc
index c2caab7..f263d82 100644
--- a/iocore/cache/CacheRead.cc
+++ b/iocore/cache/CacheRead.cc
@@ -634,13 +634,55 @@ Ldone:
Lcallreturn:
return handleEvent(AIO_EVENT_DONE, 0);
LreadMain:
- fragment++;
+ ++fragment;
doc_pos = doc->prefix_len();
+ if (req_rs.isValid()) {
+ doc_pos += req_rs.getOffset() - frag_upper_bound; // used before update!
+ }
+ frag_upper_bound += doc->data_len();
next_CacheKey(&key, &key);
SET_HANDLER(&CacheVC::openReadMain);
return openReadMain(event, e);
}
+void
+CacheVC::update_key_to_frag_idx(int target)
+{
+ if (target < fragment) { // going backwards
+ if (target < (fragment - target)) { // faster to go from start
+ fragment = 0;
+ key = earliest_key;
+ } else { // quicker to go from here to there
+ while (target < fragment) {
+ prev_CacheKey(&key, &key);
+ --fragment;
+ }
+ }
+ }
+ // advance to target if we're not already there.
+ while (target > fragment) {
+ next_CacheKey(&key, &key);
+ ++fragment;
+ }
+}
+
+int
+CacheVC::frag_idx_for_offset(HTTPInfo::FragOffset* frags, int count, uint64_t offset)
+{
+ int idx = count / 2;
+ ink_assert(offset < doc_len);
+ do {
+ uint64_t upper = idx >= count ? doc_len : frags[idx];
+ uint64_t lower = idx <= 0 ? 0 : frags[idx-1];
+ if (offset < lower) idx = idx / 2;
+ else if (offset >= upper) idx = (count + idx + 1)/2;
+ else break;
+ } while (true);
+ return idx;
+}
+
+/* There is a fragment available, decide what do to next.
+ */
int
CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
{
@@ -649,78 +691,41 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
int64_t ntodo = vio.ntodo();
int64_t bytes = doc->len - doc_pos;
IOBufferBlock *b = NULL;
- if (seek_to) { // handle do_io_pread
- if (seek_to >= doc_len) {
- vio.ndone = doc_len;
- return calluser(VC_EVENT_EOS);
- }
#ifdef HTTP_CACHE
- HTTPInfo::FragOffset* frags = alternate.get_frag_table();
- if (is_debug_tag_set("cache_seek")) {
- char b[33], c[33];
- Debug("cache_seek", "Seek @ %" PRId64" in %s from #%d @ %" PRId64"/%d:%s",
- seek_to, first_key.toHexStr(b), fragment, doc_pos, doc->len, doc->key.toHexStr(c));
- }
- /* Because single fragment objects can migrate to hang off an alt vector
- they can appear to the VC as multi-fragment when they are not really.
- The essential difference is the existence of a fragment table.
- */
- if (frags) {
- int target = 0;
- HTTPInfo::FragOffset next_off = frags[target];
- int lfi = static_cast<int>(alternate.get_frag_offset_count()) - 1;
- ink_assert(lfi >= 0); // because it's not a single frag doc.
+ if (req_rs.isValid()) {
+ int target = -1; // target fragment index.
+ uint64_t target_offset = req_rs.getOffset();
+ uint64_t lower_bound = frag_upper_bound - doc->data_len();
- /* Note: frag[i].offset is the offset of the first byte past the
- i'th fragment. So frag[0].offset is the offset of the first
- byte of fragment 1. In addition the # of fragments is one
- more than the fragment table length, the start of the last
- fragment being the last offset in the table.
- */
- if (fragment == 0 ||
- seek_to < frags[fragment-1] ||
- (fragment <= lfi && frags[fragment] <= seek_to)
- ) {
- // search from frag 0 on to find the proper frag
- while (seek_to >= next_off && target < lfi) {
- next_off = frags[++target];
- }
- if (target == lfi && seek_to >= next_off) ++target;
- } else { // shortcut if we are in the fragment already
- target = fragment;
+ if (target_offset < lower_bound || frag_upper_bound <= target_offset) {
+ HTTPInfo::FragOffset* frags = alternate.get_frag_table();
+
+ if (is_debug_tag_set("amc")) {
+ char b[33], c[33];
+ Debug("amc", "Seek @ %" PRIu64 " [r#=%d] in %s from #%d @ %" PRIu64 "/%d/%" PRId64 ":%s%s",
+ target_offset, req_rs.getIdx(), first_key.toHexStr(b), fragment, frag_upper_bound, doc->len, doc->total_len, doc->key.toHexStr(c)
+ , (frags ? "" : "no frag table")
+ );
}
- if (target != fragment) {
- // Lread will read the next fragment always, so if that
- // is the one we want, we don't need to do anything
- int cfi = fragment;
- --target;
- while (target > fragment) {
- next_CacheKey(&key, &key);
- ++fragment;
- }
- while (target < fragment) {
- prev_CacheKey(&key, &key);
- --fragment;
- }
- if (is_debug_tag_set("cache_seek")) {
- char target_key_str[33];
- key.toHexStr(target_key_str);
- Debug("cache_seek", "Seek #%d @ %" PRId64" -> #%d @ %" PRId64":%s", cfi, doc_pos, target, seek_to, target_key_str);
- }
- goto Lread;
+ if (frags) {
+ int n_frags = alternate.get_frag_offset_count();
+ target = this->frag_idx_for_offset(frags, n_frags, target_offset);
+ this->update_key_to_frag_idx(target);
+ /// one frag short, because it gets bumped when the fragment is actually read.
+ frag_upper_bound = target > 0 ? frags[target-1] : 0;
+ } else { // we don't support non-monotonic forward requests on objects with no frag table.
+ ink_release_assert(target_offset >= frag_upper_bound);
}
- }
- doc_pos = doc->prefix_len() + seek_to;
- if (fragment) doc_pos -= static_cast<int64_t>(frags[fragment-1]);
- vio.ndone = 0;
- seek_to = 0;
- ntodo = vio.ntodo();
- bytes = doc->len - doc_pos;
- if (is_debug_tag_set("cache_seek")) {
- char target_key_str[33];
- key.toHexStr(target_key_str);
- Debug("cache_seek", "Read # %d @ %" PRId64"/%d for %" PRId64, fragment, doc_pos, doc->len, bytes);
+ goto Lread;
+ } else {
+ // Adjust fragment starting offset to account for target offset.
+ int r_doc_pos = doc->prefix_len() + (target_offset - lower_bound);
+ if (doc_pos < r_doc_pos) {
+ doc_pos = r_doc_pos;
+ bytes = doc->len - doc_pos;
+ }
+ bytes = std::min(bytes, static_cast<int64_t>(req_rs.getRemnantSize()));
}
#endif
}
@@ -730,6 +735,7 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
return EVENT_CONT;
if ((bytes <= 0) && vio.ntodo() >= 0)
goto Lread;
+ // Do the write to downstream consumers.
if (bytes > vio.ntodo())
bytes = vio.ntodo();
b = new_IOBufferBlock(buf, bytes, doc_pos);
@@ -737,6 +743,7 @@ CacheVC::openReadMain(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */)
vio.buffer.writer()->append_block(b);
vio.ndone += bytes;
doc_pos += bytes;
+ req_rs.consume(bytes);
if (vio.ntodo() <= 0)
return calluser(VC_EVENT_READ_COMPLETE);
else {
@@ -857,6 +864,7 @@ CacheVC::openReadStartEarliest(int /* event ATS_UNUSED */, Event * /* e ATS_UNUS
earliest_key = key;
doc_pos = doc->prefix_len();
next_CacheKey(&key, &doc->key);
+ fragment = 1;
vol->begin_read(this);
if (vol->within_hit_evacuate_window(&earliest_dir) &&
(!cache_config_hit_evacuate_size_limit || doc_len <= (uint64_t)cache_config_hit_evacuate_size_limit)
@@ -1016,6 +1024,10 @@ Lrestart:
/*
This code follows CacheVC::openReadStartEarliest closely,
if you change this you might have to change that.
+
+ This handles the I/O completion of reading the first doc of the object.
+ If there are alternates, we chain to openreadStartEarliest to read the
+ earliest doc.
*/
int
CacheVC::openReadStartHead(int event, Event * e)
@@ -1144,6 +1156,8 @@ CacheVC::openReadStartHead(int event, Event * e)
ink_assert(doc->hlen);
doc_pos = doc->prefix_len();
next_CacheKey(&key, &doc->key);
+ fragment = 1;
+ frag_upper_bound = doc->data_len();
} else {
f.single_fragment = false;
}
@@ -1151,6 +1165,8 @@ CacheVC::openReadStartHead(int event, Event * e)
#endif
{
next_CacheKey(&key, &doc->key);
+ fragment = 1;
+ frag_upper_bound = doc->data_len();
f.single_fragment = doc->single_fragment();
doc_pos = doc->prefix_len();
doc_len = doc->total_len;
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d33b6255/iocore/cache/I_CacheDefs.h
----------------------------------------------------------------------
diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h
index df430f4..ce50958 100644
--- a/iocore/cache/I_CacheDefs.h
+++ b/iocore/cache/I_CacheDefs.h
@@ -203,9 +203,18 @@ struct RangeSpec {
*/
size_t count() const;
- /// If this is a valid, single range specification.
+ /// If this is a valid single range specification.
bool isSingle() const;
+ /// If this is a valid multi range specification.
+ bool isMulti() const;
+
+ /// Test if this contains at least one valid range.
+ bool isValid() const;
+
+ /// Test if this is a valid but empty range spec.
+ bool isEmpty() const;
+
protected:
self& add(uint64_t low, uint64_t high);
};
@@ -221,12 +230,30 @@ RangeSpec::isSingle() const
return SINGLE == _state;
}
+inline bool
+RangeSpec::isMulti() const
+{
+ return MULTI == _state;
+}
+
+inline bool
+RangeSpec::isEmpty() const
+{
+ return EMPTY == _state;
+}
+
inline size_t
RangeSpec::count() const
{
return SINGLE == _state ? 1 : _ranges.size();
}
+inline bool
+RangeSpec::isValid() const
+{
+ return SINGLE == _state || MULTI == _state;
+}
+
inline RangeSpec::Range&
RangeSpec::Range::invalidate()
{
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/d33b6255/iocore/cache/P_CacheInternal.h
----------------------------------------------------------------------
diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h
index 5a0b8fc..2de18a3 100644
--- a/iocore/cache/P_CacheInternal.h
+++ b/iocore/cache/P_CacheInternal.h
@@ -235,20 +235,43 @@ extern int good_interim_disks;
/** Range operation tracking.
- This holds a range specification and tracks where in the range the current
- cache operation is.
+ This holds a range specification. It also tracks the current object offset and the individual range.
+
+ For simplification of the logic that uses this class it will pretend to be a single range of
+ the object size if it is empty. To return the correct response we still need to distinuish
+ those two cases.
*/
class CacheRange : public RangeSpec
{
public:
typedef CacheRange self; ///< Self reference type.
+ typedef RangeSpec super; ///< Parent type.
/// Default constructor
- CacheRange() : _offset(0), _idx(-1) { }
+ CacheRange() : super(), _offset(0), _idx(-1) { }
+
+ /// Convert to specific values based on content @a length.
+ bool finalize(uint64_t length);
+
+ /// Get the current object offset
+ uint64_t getOffset() const;
+
+ /// Get the current range index.
+ int getIdx() const;
+
+ /// Get the remaining contiguous bytes for the current range.
+ uint64_t getRemnantSize() const;
+
+ /** Advance @a size bytes in the range spec.
+
+ @return The resulting offset in the object.
+ */
+ uint64_t consume(uint64_t size);
protected:
- uint64_t _offset; ///< Current offset into the range.
- int _idx; ///< Current range index. (< 0 -> not in a range)
+ uint64_t _len; ///< Total object length.
+ uint64_t _offset; ///< Offset in content.
+ int _idx; ///< Current range index. (< 0 means not in a range)
};
// CacheVC
@@ -384,6 +407,13 @@ struct CacheVC: public CacheVConnection
@return Length of header data used for alternates.
*/
virtual uint32_t load_http_info(CacheHTTPInfoVector* info, struct Doc* doc, RefCountObj * block_ptr = NULL);
+
+ /// Change member @a key to be the key for the @a target 'th fragment.
+ void update_key_to_frag_idx(int target);
+ /// Compute the index of the fragment that contains the byte at content @a offset.
+ /// The table of @a frags and its @a count must be provided.
+ int frag_idx_for_offset(HTTPInfo::FragOffset* frags, int count, uint64_t offset);
+
#endif
virtual bool is_pread_capable();
virtual bool set_pin_in_cache(time_t time_pin);
@@ -473,6 +503,11 @@ struct CacheVC: public CacheVConnection
uint64_t total_len; // total length written and available to write
uint64_t doc_len; // total_length (of the selected alternate for HTTP)
uint64_t update_len;
+ /// The offset in the content of the first byte beyond the end of the current fragment.
+ /// @internal This seems very weird but I couldn't figure out how to keep the more sensible
+ /// lower bound correctly updated.
+ /// The lower bound can can computed by subtracting doc->len from this value.
+ uint64_t frag_upper_bound;
int fragment;
int scan_msec_delay;
CacheVC *write_vc;
@@ -1384,6 +1419,29 @@ local_cache()
return theCache;
}
+inline uint64_t
+CacheRange::getOffset() const
+{
+ return _offset;
+}
+
+inline int
+CacheRange::getIdx() const
+{
+ return _idx;
+}
+
+inline uint64_t
+CacheRange::getRemnantSize() const
+{
+ uint64_t zret = 0;
+ if (this->isEmpty()) zret = _len - _offset;
+ else if (this->isSingle()) zret = (_single._max - _offset) + 1;
+ else if (this->isMulti() && 0 <= _idx && _idx < static_cast<int>(_ranges.size()))
+ zret = (_ranges[_idx]._max - _offset) + 1;
+ return zret;
+}
+
LINK_DEFINITION(CacheVC, opendir_link)
#endif /* _P_CACHE_INTERNAL_H__ */
[2/8] trafficserver git commit: before seek change.
Posted by am...@apache.org.
before seek change.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/ee708499
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/ee708499
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/ee708499
Branch: refs/heads/ts-974-multi-range-read
Commit: ee7084990274939458314e35157a23abb13f3edc
Parents: 50f37f3
Author: Alan M. Carroll <so...@yahoo-inc.com>
Authored: Wed Nov 26 09:04:13 2014 -0600
Committer: Alan M. Carroll <so...@yahoo-inc.com>
Committed: Sat Dec 6 11:56:03 2014 -0600
----------------------------------------------------------------------
iocore/cache/CacheHttp.cc | 39 ++++++++++++++---
iocore/cache/CacheRead.cc | 22 ++++++----
iocore/cache/I_CacheDefs.h | 87 ++++++++++++++++++++++++++++++++++---
iocore/cache/P_CacheInternal.h | 27 ++++++++++++
lib/ts/InkErrno.h | 1 +
5 files changed, 155 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/ee708499/iocore/cache/CacheHttp.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheHttp.cc b/iocore/cache/CacheHttp.cc
index e58e42e..024f5ca 100644
--- a/iocore/cache/CacheHttp.cc
+++ b/iocore/cache/CacheHttp.cc
@@ -329,20 +329,47 @@ RangeSpec::parse(char const* v, int len)
RangeSpec&
RangeSpec::add(uint64_t low, uint64_t high)
{
- if (EMPTY == _state || INVALID == _state) {
+ if (MULTI == _state) {
+ _ranges.push_back(Range(low, high));
+ } else if (SINGLE == _state) {
+ _ranges.push_back(_single);
+ _ranges.push_back(Range(low,high));
+ _state = MULTI;
+ } else {
_single._min = low;
_single._max = high;
_state = SINGLE;
- } else {
- _ranges.push_back(Range(low, high));
}
return *this;
}
-size_t
-RangeSpec::count() const
+bool
+RangeSpec::finalize(uint64_t len)
{
- return (EMPTY == _state || INVALID == _state) ? 0 : 1 + (_ranges.size());
+ if (INVALID == _state || EMPTY == _state) {
+ // nothing but simplifying later logic.
+ } else if (0 == len) {
+ /* Must special case zero length content
+ - suffix ranges are OK but other ranges are not.
+ - SM must return a 200 (not 206 or 416) for a valid range on zero length content.
+ (this is what Apache HTTPD does and seems the least bad thing)
+ - Therefore we don't bother actually adjusting the ranges as values don't matter.
+ */
+ if (!_single.isSuffix()) _state = INVALID;
+ if (MULTI == _state) {
+ for ( RangeBox::iterator spot = _ranges.begin(), limit = _ranges.end() ; spot != limit && MULTI == _state ; ++spot ) {
+ if (!spot->isSuffix()) _state = INVALID;
+ }
+ }
+ } else { // len > 0
+ if (!_single.finalize(len)) _state = INVALID;
+ if (MULTI == _state) {
+ for ( RangeBox::iterator spot = _ranges.begin(), limit = _ranges.end() ; spot != limit && MULTI == _state; ++spot ) {
+ if (!spot->finalize(len)) _state = INVALID;
+ }
+ }
+ }
+ return INVALID != _state;
}
/*-------------------------------------------------------------------------
-------------------------------------------------------------------------*/
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/ee708499/iocore/cache/CacheRead.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc
index 944cb37..c2caab7 100644
--- a/iocore/cache/CacheRead.cc
+++ b/iocore/cache/CacheRead.cc
@@ -106,18 +106,11 @@ Cache::open_read(Continuation * cont, CacheKey * key, CacheHTTPHdr * request,
OpenDirEntry *od = NULL;
CacheVC *c = NULL;
- MIMEField* range_field = request->field_find(MIME_FIELD_RANGE, MIME_LEN_RANGE);
- if (range_field) {
- RangeSpec rs;
- char const* value;
- int len;
- value = range_field->value_get(&len);
- rs.parse(value, len);
- }
-
{
CACHE_TRY_LOCK(lock, vol->mutex, mutex->thread_holding);
if (!lock.is_locked() || (od = vol->open_read(key)) || dir_probe(key, vol, &result, &last_collision)) {
+ MIMEField* range_field = request->field_find(MIME_FIELD_RANGE, MIME_LEN_RANGE);
+
c = new_CacheVC(cont);
c->first_key = c->key = c->earliest_key = *key;
c->vol = vol;
@@ -128,6 +121,12 @@ Cache::open_read(Continuation * cont, CacheKey * key, CacheHTTPHdr * request,
c->frag_type = CACHE_FRAG_TYPE_HTTP;
c->params = params;
c->od = od;
+ if (range_field) {
+ char const* value;
+ int len;
+ value = range_field->value_get(&len);
+ c->req_rs.parse(value, len);
+ }
}
if (!lock.is_locked()) {
SET_CONTINUATION_HANDLER(c, &CacheVC::openReadStartHead);
@@ -1157,6 +1156,11 @@ CacheVC::openReadStartHead(int event, Event * e)
doc_len = doc->total_len;
}
+ if (!req_rs.finalize(doc_len)) {
+ err = ECACHE_BAD_REQUEST_RANGE;
+ goto Ldone;
+ }
+
if (is_debug_tag_set("cache_read")) { // amc debug
char xt[33],yt[33];
Debug("cache_read", "CacheReadStartHead - read %s target %s - %s %d of %" PRId64" bytes, %d fragments",
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/ee708499/iocore/cache/I_CacheDefs.h
----------------------------------------------------------------------
diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h
index 2c5caa4..df430f4 100644
--- a/iocore/cache/I_CacheDefs.h
+++ b/iocore/cache/I_CacheDefs.h
@@ -154,15 +154,22 @@ struct RangeSpec {
/// Construct as the range ( @a low .. @a high )
Range(uint64_t low, uint64_t high) : _min(low), _max(high) {}
- bool isValid() const { return _min <= _max || (0 == _max && 0 < _min); }
+ /// Test if this range is a trailing (terminal) range.
+ bool isSuffix() const;
+ /// Test if this range is a valid range.
+ bool isValid() const;
+ /// Adjust the range values based on content size @a len.
+ bool finalize(uint64_t len);
+ /// Force the range to an invalid state.
+ Range& invalidate();
};
/// Current state of the overall specification.
/// @internal We can distinguish between @c SINGLE and @c MULTI by looking at the
/// size of @a _ranges but we need this to mark @c EMPTY vs. not.
enum State {
- INVALID, ///< Range parsing failed.
EMPTY, ///< No range.
+ INVALID, ///< Range parsing failed.
SINGLE, ///< Single range.
MULTI, ///< Multiple ranges.
} _state;
@@ -171,21 +178,89 @@ struct RangeSpec {
/// By separating this out we can avoid allocation in the case of a single
/// range value, which is by far the most common ( > 99% in my experience).
Range _single;
- /// Storage for range values past the first one.
- std::vector<Range> _ranges;
+ /// Storage for range values.
+ typedef std::vector<Range> RangeBox;
+ /// The first range is copied here if there is more than one (to simplify).
+ RangeBox _ranges;
- RangeSpec() : _state(EMPTY)
- {}
+ /// Default constructor - invalid range
+ RangeSpec();
/** Parse a range field and update @a this with the results.
@return @c true if @a v was a valid range specifier, @c false otherwise.
*/
bool parse(char const* v, int len);
+ /** Validate and convert for a specific content @a length.
+
+ @return @c true if the range is satisfiable per the HTTP spec, @c false otherwise.
+ Note a range spec with no ranges is always satisfiable.
+ */
+ bool finalize(uint64_t length);
+
+ /** Number of distinct ranges.
+ @return Number of ranges.
+ */
size_t count() const;
+ /// If this is a valid, single range specification.
+ bool isSingle() const;
+
protected:
self& add(uint64_t low, uint64_t high);
};
+inline
+RangeSpec::RangeSpec() : _state(EMPTY)
+{
+}
+
+inline bool
+RangeSpec::isSingle() const
+{
+ return SINGLE == _state;
+}
+
+inline size_t
+RangeSpec::count() const
+{
+ return SINGLE == _state ? 1 : _ranges.size();
+}
+
+inline RangeSpec::Range&
+RangeSpec::Range::invalidate()
+{
+ _min = UINT64_MAX;
+ _max = 1;
+ return *this;
+}
+
+inline bool
+RangeSpec::Range::isSuffix() const
+{
+ return 0 == _max && _min > 0;
+}
+
+inline bool
+RangeSpec::Range::isValid() const
+{
+ return _min <= _max || this->isSuffix();
+}
+
+inline bool
+RangeSpec::Range::finalize(uint64_t len)
+{
+ ink_assert(len > 0);
+ bool zret = true; // is this range satisfiable for @a len?
+ if (this->isSuffix()) {
+ _max = len - 1;
+ _min = _min > len ? 0 : len - _min;
+ } else if (_min < len) {
+ _max = MIN(_max,len);
+ } else {
+ this->invalidate();
+ zret = false;
+ }
+ return zret;
+}
#endif // __CACHE_DEFS_H__
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/ee708499/iocore/cache/P_CacheInternal.h
----------------------------------------------------------------------
diff --git a/iocore/cache/P_CacheInternal.h b/iocore/cache/P_CacheInternal.h
index 57c5b0b..5a0b8fc 100644
--- a/iocore/cache/P_CacheInternal.h
+++ b/iocore/cache/P_CacheInternal.h
@@ -232,6 +232,25 @@ extern int cache_config_mutex_retry_delay;
#if TS_USE_INTERIM_CACHE == 1
extern int good_interim_disks;
#endif
+
+/** Range operation tracking.
+
+ This holds a range specification and tracks where in the range the current
+ cache operation is.
+*/
+class CacheRange : public RangeSpec
+{
+ public:
+ typedef CacheRange self; ///< Self reference type.
+
+ /// Default constructor
+ CacheRange() : _offset(0), _idx(-1) { }
+
+ protected:
+ uint64_t _offset; ///< Current offset into the range.
+ int _idx; ///< Current range index. (< 0 -> not in a range)
+};
+
// CacheVC
struct CacheVC: public CacheVConnection
{
@@ -462,6 +481,12 @@ struct CacheVC: public CacheVConnection
int header_to_write_len;
void *header_to_write;
short writer_lock_retry;
+ /* Range specs for range based operations.
+ @a req_rng is the range spec in the request from the User Agent.
+ @a rsp_rng is the range spec sent to the origin server.
+ */
+ CacheRange req_rs;
+ CacheRange rsp_rs;
#if TS_USE_INTERIM_CACHE == 1
InterimCacheVol *interim_vol;
MigrateToInterimCache *mts;
@@ -628,6 +653,8 @@ free_CacheVC(CacheVC *cont)
cont->alternate_index = CACHE_ALT_INDEX_DEFAULT;
if (cont->scan_vol_map)
ats_free(cont->scan_vol_map);
+ cont->req_rs.~CacheRange();
+ cont->rsp_rs.~CacheRange();
memset((char *) &cont->vio, 0, cont->size_to_init);
#ifdef CACHE_STAT_PAGES
ink_assert(!cont->stat_link.next && !cont->stat_link.prev);
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/ee708499/lib/ts/InkErrno.h
----------------------------------------------------------------------
diff --git a/lib/ts/InkErrno.h b/lib/ts/InkErrno.h
index 7f8676e..bb9bfb1 100644
--- a/lib/ts/InkErrno.h
+++ b/lib/ts/InkErrno.h
@@ -66,6 +66,7 @@
#define ECACHE_NOT_READY (CACHE_ERRNO+7)
#define ECACHE_ALT_MISS (CACHE_ERRNO+8)
#define ECACHE_BAD_READ_REQUEST (CACHE_ERRNO+9)
+#define ECACHE_BAD_REQUEST_RANGE (CACHE_ERRNO+10)
#define EHTTP_ERROR (HTTP_ERRNO+0)
[3/8] trafficserver git commit: Parsing ranges.
Posted by am...@apache.org.
Parsing ranges.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/50f37f3a
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/50f37f3a
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/50f37f3a
Branch: refs/heads/ts-974-multi-range-read
Commit: 50f37f3a7d73c01d909c099e9b9ef70dc1c5f339
Parents: 45dc0c6
Author: Alan M. Carroll <so...@yahoo-inc.com>
Authored: Sat Nov 22 17:56:47 2014 -0600
Committer: Alan M. Carroll <so...@yahoo-inc.com>
Committed: Sat Dec 6 11:56:03 2014 -0600
----------------------------------------------------------------------
iocore/cache/CacheHttp.cc | 66 ++++++++++++++++++++++++++++++++++++++++-
iocore/cache/CacheRead.cc | 9 ++++++
iocore/cache/I_CacheDefs.h | 28 +++++++++++++----
lib/ts/ParseRules.cc | 17 +++++++++++
lib/ts/ParseRules.h | 20 +++++++++++++
5 files changed, 134 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/50f37f3a/iocore/cache/CacheHttp.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheHttp.cc b/iocore/cache/CacheHttp.cc
index 31531f2..e58e42e 100644
--- a/iocore/cache/CacheHttp.cc
+++ b/iocore/cache/CacheHttp.cc
@@ -261,24 +261,88 @@ CacheHTTPInfoVector::get_handles(const char *buf, int length, RefCountObj * bloc
bool
RangeSpec::parse(char const* v, int len)
{
+ // Maximum # of digits permitted for an offset. Avoid issues with overflow.
+ static size_t const MAX_DIGITS = 15;
char const PREFIX[] = { 'b', 'y', 't', 'e', 's', '=' };
ts::ConstBuffer src(v, len);
+ size_t n;
+ _state = EMPTY;
src.skip(&ParseRules::is_ws);
if (src.size() > sizeof(PREFIX) && 0 == memcmp(src.data(), PREFIX, sizeof(PREFIX))) {
+ _state = INVALID; // something, it needs to be correct.
src += sizeof(PREFIX);
while (src) {
ts::ConstBuffer max = src.splitOn(',');
+
+ if (!max) { // no comma so everything in @a src should be processed as a single range.
+ max = src;
+ src.reset();
+ }
+
ts::ConstBuffer min = max.splitOn('-');
+
src.skip(&ParseRules::is_ws);
+ // Spec forbids whitspace anywhere in the range element.
if (min) {
+ if (ParseRules::is_digit(*min) && min.size() <= MAX_DIGITS) {
+ uint64_t low = ats_strto64(min.data(), min.size(), &n);
+ if (n < min.size()) break; // extra cruft in range, not even ws allowed
+ if (max) {
+ if (ParseRules::is_digit(*max) && max.size() <= MAX_DIGITS) {
+ uint64_t high = ats_strto64(max.data(), max.size(), &n);
+ if (n < max.size() && (max += n).skip(&ParseRules::is_ws))
+ break; // non-ws cruft after maximum
+ else
+ this->add(low, high);
+ } else {
+ break; // invalid characters for maximum
+ }
+ } else {
+ this->add(low, UINT64_MAX); // "X-" : "offset X to end of content"
+ }
+ } else {
+ break; // invalid characters for minimum
+ }
} else {
+ if (max) {
+ if (ParseRules::is_digit(*max) && max.size() <= MAX_DIGITS) {
+ uint64_t high = ats_strto64(max.data(), max.size(), &n);
+ if (n < max.size() && (max += n).skip(&ParseRules::is_ws)) {
+ break; // cruft after end of maximum
+ } else {
+ this->add(high, 0);
+ }
+ } else {
+ break; // invalid maximum
+ }
+ }
}
}
+ if (src) _state = INVALID; // didn't parse everything, must have been an error.
}
- return true;
+ return _state != INVALID;
+}
+
+RangeSpec&
+RangeSpec::add(uint64_t low, uint64_t high)
+{
+ if (EMPTY == _state || INVALID == _state) {
+ _single._min = low;
+ _single._max = high;
+ _state = SINGLE;
+ } else {
+ _ranges.push_back(Range(low, high));
+ }
+ return *this;
+}
+
+size_t
+RangeSpec::count() const
+{
+ return (EMPTY == _state || INVALID == _state) ? 0 : 1 + (_ranges.size());
}
/*-------------------------------------------------------------------------
-------------------------------------------------------------------------*/
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/50f37f3a/iocore/cache/CacheRead.cc
----------------------------------------------------------------------
diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc
index d6fe0f8..944cb37 100644
--- a/iocore/cache/CacheRead.cc
+++ b/iocore/cache/CacheRead.cc
@@ -106,6 +106,15 @@ Cache::open_read(Continuation * cont, CacheKey * key, CacheHTTPHdr * request,
OpenDirEntry *od = NULL;
CacheVC *c = NULL;
+ MIMEField* range_field = request->field_find(MIME_FIELD_RANGE, MIME_LEN_RANGE);
+ if (range_field) {
+ RangeSpec rs;
+ char const* value;
+ int len;
+ value = range_field->value_get(&len);
+ rs.parse(value, len);
+ }
+
{
CACHE_TRY_LOCK(lock, vol->mutex, mutex->thread_holding);
if (!lock.is_locked() || (od = vol->open_read(key)) || dir_probe(key, vol, &result, &last_collision)) {
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/50f37f3a/iocore/cache/I_CacheDefs.h
----------------------------------------------------------------------
diff --git a/iocore/cache/I_CacheDefs.h b/iocore/cache/I_CacheDefs.h
index 84887de..2c5caa4 100644
--- a/iocore/cache/I_CacheDefs.h
+++ b/iocore/cache/I_CacheDefs.h
@@ -138,17 +138,30 @@ typedef CryptoHash CacheKey;
This represents the data for an HTTP range specification.
*/
struct RangeSpec {
+ typedef RangeSpec self;
+
/** A range of bytes in an object.
- If @a _min > @a _max this means the range is backwards and counts from the
- end of the object. That is (1,0) means the last byte of content.
+ If @a _min > 0 and @a _max == 0 the range is backwards and counts from the
+ end of the object. That is (100,0) means the last 100 bytes of content.
*/
- struct Descriptor {
+ struct Range {
uint64_t _min;
uint64_t _max;
+
+ /// Default constructor - invalid range.
+ Range() : _min(UINT64_MAX), _max(1) { }
+ /// Construct as the range ( @a low .. @a high )
+ Range(uint64_t low, uint64_t high) : _min(low), _max(high) {}
+
+ bool isValid() const { return _min <= _max || (0 == _max && 0 < _min); }
};
+ /// Current state of the overall specification.
+ /// @internal We can distinguish between @c SINGLE and @c MULTI by looking at the
+ /// size of @a _ranges but we need this to mark @c EMPTY vs. not.
enum State {
+ INVALID, ///< Range parsing failed.
EMPTY, ///< No range.
SINGLE, ///< Single range.
MULTI, ///< Multiple ranges.
@@ -157,9 +170,9 @@ struct RangeSpec {
/// The first range value.
/// By separating this out we can avoid allocation in the case of a single
/// range value, which is by far the most common ( > 99% in my experience).
- Descriptor _singleton;
+ Range _single;
/// Storage for range values past the first one.
- std::vector<Descriptor> _ranges;
+ std::vector<Range> _ranges;
RangeSpec() : _state(EMPTY)
{}
@@ -168,6 +181,11 @@ struct RangeSpec {
@return @c true if @a v was a valid range specifier, @c false otherwise.
*/
bool parse(char const* v, int len);
+
+ size_t count() const;
+
+protected:
+ self& add(uint64_t low, uint64_t high);
};
#endif // __CACHE_DEFS_H__
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/50f37f3a/lib/ts/ParseRules.cc
----------------------------------------------------------------------
diff --git a/lib/ts/ParseRules.cc b/lib/ts/ParseRules.cc
index 1834bbd..38803bb 100644
--- a/lib/ts/ParseRules.cc
+++ b/lib/ts/ParseRules.cc
@@ -307,3 +307,20 @@ ink_atoi64(const char *str, int len)
}
return num;
}
+
+uint64_t ats_strto64(char const* s, size_t len, size_t* used)
+{
+ uint64_t zret = 0;
+ char const* spot = s;
+
+ if (s && len) {
+ for (char const* limit = spot + len ; spot < limit && ParseRules::is_digit(*spot) ; ++spot) {
+ zret *= 10;
+ zret += *spot - '0';
+ }
+ }
+
+ if (used) *used = spot - s;
+ return zret;
+
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/50f37f3a/lib/ts/ParseRules.h
----------------------------------------------------------------------
diff --git a/lib/ts/ParseRules.h b/lib/ts/ParseRules.h
index 6bc7483..39db30a 100644
--- a/lib/ts/ParseRules.h
+++ b/lib/ts/ParseRules.h
@@ -845,4 +845,24 @@ ink_atoui(const char *str)
return static_cast<int>(val);
}
+/** Convert a span of characters to an unsigned 64 bit value.
+
+ Parsing starts at @a s and continues for at most @a len characters.
+ Parsing stops when the first non-digit is encountered. Leading whitespace is not permitted.
+ @a *used is set to the number of characters parsed if @a used is not @c NULL.
+ If @a *used is set to 0 and 0 is returned, then no characters were parsed.
+
+ Key features
+
+ - No termination required.
+ - Number of parsed characters returned.
+ - Unsigned 64 bit return.
+ - Clip to UINT64_MAX;
+
+ @return The binary equivalent of @a s.
+
+ @internal All of these conversions and none work as I need. Sigh.
+*/
+uint64_t ats_strto64(char const* s, size_t len, size_t* used);
+
#endif /* #if !defined (_ParseRules_h_) */