You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by os...@apache.org on 2014/08/20 22:57:46 UTC
[6/7] ats_pagespeed: rename ats_speed -> ats_pagespeed
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_pagespeed.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_pagespeed.cc b/plugins/experimental/ats_pagespeed/ats_pagespeed.cc
new file mode 100644
index 0000000..4f6cea5
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_pagespeed.cc
@@ -0,0 +1,1093 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+// TODO(oschaaf): remove what isn't used
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS
+#endif
+#include <stdlib.h>
+#include <errno.h>
+#include <stdio.h>
+#include <limits.h>
+#include <stdint.h>
+#include <sys/inotify.h>
+#include <unistd.h>
+
+#include <ts/ts.h>
+
+#include <vector>
+#include <set>
+
+
+#include "ats_pagespeed.h"
+
+#include "ats_config.h"
+#include "ats_header_utils.h"
+#include "ats_rewrite_options.h"
+#include "ats_log_message_handler.h"
+
+#include "base/logging.h"
+#include "net/instaweb/http/public/response_headers.h"
+#include "net/instaweb/util/public/string_util.h"
+
+#include "ats_base_fetch.h"
+#include "ats_resource_intercept.h"
+#include "ats_beacon_intercept.h"
+#include "ats_process_context.h"
+#include "ats_rewrite_driver_factory.h"
+#include "ats_rewrite_options.h"
+#include "ats_server_context.h"
+
+#include "net/instaweb/rewriter/public/rewrite_stats.h"
+#include "net/instaweb/system/public/in_place_resource_recorder.h"
+
+#include "net/instaweb/automatic/public/proxy_fetch.h"
+#include "net/instaweb/http/public/content_type.h"
+#include "net/instaweb/http/public/request_context.h"
+#include "net/instaweb/rewriter/public/experiment_matcher.h"
+#include "net/instaweb/rewriter/public/experiment_util.h"
+#include "net/instaweb/rewriter/public/process_context.h"
+#include "net/instaweb/rewriter/public/resource_fetch.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_query.h"
+#include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/public/global_constants.h"
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/google_url.h"
+#include "net/instaweb/util/public/gzip_inflater.h"
+#include "net/instaweb/util/public/query_params.h"
+#include "net/instaweb/util/public/statistics_logger.h"
+#include "net/instaweb/util/public/stdio_file_system.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_writer.h"
+#include "net/instaweb/util/public/time_util.h"
+#include "net/instaweb/util/stack_buffer.h"
+#include "net/instaweb/system/public/system_request_context.h"
+
+
+#include <dirent.h>
+
+using namespace net_instaweb;
+
+static AtsProcessContext* ats_process_context;
+static const char* DEBUG_TAG = "ats_pagespeed_transform";
+static int TXN_INDEX_ARG;
+static int TXN_INDEX_OWNED_ARG;
+static int TXN_INDEX_OWNED_ARG_SET;
+static int TXN_INDEX_OWNED_ARG_UNSET;
+TSMutex config_mutex = TSMutexCreate();
+AtsConfig* config = NULL;
+TransformCtx* get_transaction_context(TSHttpTxn txnp) {
+ return (TransformCtx *) TSHttpTxnArgGet(txnp, TXN_INDEX_ARG);
+}
+
+static TransformCtx *
+ats_ctx_alloc()
+{
+ TransformCtx *ctx;
+
+ ctx = (TransformCtx *) TSmalloc(sizeof(TransformCtx));
+ ctx->downstream_vio = NULL;
+ ctx->downstream_buffer = NULL;
+ ctx->downstream_length = 0;
+ ctx->state = transform_state_initialized;
+
+ ctx->base_fetch = NULL;
+ ctx->proxy_fetch = NULL;
+
+ ctx->inflater = NULL;
+ ctx->url_string = NULL;
+ ctx->gurl = NULL;
+ ctx->write_pending = false;
+ ctx->fetch_done = false;
+ ctx->resource_request = false;
+ ctx->beacon_request = false;
+ ctx->transform_added = false;
+ ctx->mps_user_agent = false;
+ ctx->user_agent = NULL;
+ ctx->server_context = NULL;
+ ctx->html_rewrite = false;
+ ctx->request_method = NULL;
+ ctx->alive = 0xaaaa;
+ ctx->options = NULL;
+ ctx->to_host = NULL;
+ return ctx;
+}
+
+void
+ats_ctx_destroy(TransformCtx * ctx)
+{
+ TSReleaseAssert(ctx);
+ CHECK(ctx->alive == 0xaaaa) << "Already dead!";
+ ctx->alive = 0xbbbb;
+
+ if (ctx->base_fetch != NULL) {
+ ctx->base_fetch->Release();
+ ctx->base_fetch = NULL;
+ }
+
+ if (ctx->proxy_fetch != NULL) {
+ ctx->proxy_fetch->Done(false /* failure */);
+ ctx->proxy_fetch = NULL;
+ }
+
+ if (ctx->inflater != NULL) {
+ delete ctx->inflater;
+ ctx->inflater = NULL;
+ }
+
+ if (ctx->downstream_buffer) {
+ TSIOBufferDestroy(ctx->downstream_buffer);
+ }
+
+ if (ctx->url_string != NULL) {
+ delete ctx->url_string;
+ ctx->url_string = NULL;
+ }
+
+ if (ctx->gurl != NULL) {
+ delete ctx->gurl;
+ ctx->gurl = NULL;
+ }
+ if (ctx->user_agent != NULL) {
+ delete ctx->user_agent;
+ ctx->user_agent = NULL;
+ }
+ ctx->request_method = NULL;
+ if (ctx->options != NULL) {
+ delete ctx->options;
+ ctx->options = NULL;
+ }
+ if (ctx->to_host != NULL) {
+ delete ctx->to_host;
+ ctx->to_host = NULL;
+ }
+ TSfree(ctx);
+}
+
+RewriteOptions* ps_determine_request_options(
+ ServerContext* server_context,
+ RequestHeaders* request_headers,
+ ResponseHeaders* response_headers,
+ GoogleUrl* url) {
+ // Stripping ModPagespeed query params before the property cache lookup to
+ // make cache key consistent for both lookup and storing in cache.
+ //
+ // Sets option from request headers and url.
+ RewriteQuery rewrite_query;
+ if (!server_context->GetQueryOptions(url, request_headers,
+ response_headers, &rewrite_query)) {
+ // Failed to parse query params or request headers. Treat this as if there
+ // were no query params given.
+ TSError("ps_route rerquest: parsing headers or query params failed.");
+ return NULL;
+ }
+
+ // Will be NULL if there aren't any options set with query params or in
+ // headers.
+ return rewrite_query.ReleaseOptions();
+}
+
+bool ps_determine_options(ServerContext* server_context,
+ // Directory-specific options, usually null. They've already been rebased off
+ // of the global options as part of the configuration process.
+ RewriteOptions* directory_options,
+ RequestHeaders* request_headers,
+ ResponseHeaders* response_headers,
+ RewriteOptions** options,
+ GoogleUrl* url) {
+ // Global options for this server. Never null.
+ RewriteOptions* global_options = server_context->global_options();
+
+ // Request-specific options, nearly always null. If set they need to be
+ // rebased on the directory options or the global options.
+ RewriteOptions* request_options = ps_determine_request_options(
+ server_context, request_headers, response_headers, url);
+
+ // Because the caller takes ownership of any options we return, the only
+ // situation in which we can avoid allocating a new RewriteOptions is if the
+ // global options are ok as are.
+ if (directory_options == NULL && request_options == NULL &&
+ !global_options->running_experiment()) {
+ return true;
+ }
+
+ // Start with directory options if we have them, otherwise request options.
+ if (directory_options != NULL) {
+ //*options = directory_options->Clone();
+ // OS: HACK! TODO!
+ *options = global_options->Clone();
+ (*options)->Merge(*directory_options);
+ } else {
+ *options = global_options->Clone();
+ }
+
+ // Modify our options in response to request options or experiment settings,
+ // if we need to. If there are request options then ignore the experiment
+ // because we don't want experiments to be contaminated with unexpected
+ // settings.
+ if (request_options != NULL) {
+ (*options)->Merge(*request_options);
+ delete request_options;
+ }
+ // TODO(oschaaf): experiments
+ /*else if ((*options)->running_experiment()) {
+ bool ok = ps_set_experiment_state_and_cookie(
+ r, request_headers, *options, url->Host());
+ if (!ok) {
+ delete *options;
+ *options = NULL;
+ return false;
+ }
+ }*/
+
+ return true;
+}
+
+void
+handle_send_response_headers(TSHttpTxn txnp) {
+ TransformCtx* ctx = get_transaction_context(txnp);
+ // TODO(oschaaf): Fix the response headers!!
+ bool is_owned = TSHttpTxnArgGet(txnp, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET;
+ if (!is_owned) {
+ return;
+ }
+ CHECK(ctx->alive == 0xaaaa) << "Already dead !";
+ if (ctx->html_rewrite) {
+ TSMBuffer bufp = NULL;
+ TSMLoc hdr_loc = NULL;
+ if (ctx->base_fetch == NULL) {
+ // TODO(oschaaf): figure out when this happens.
+ return;
+ }
+
+ if (TSHttpTxnClientRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) {
+ ResponseHeaders* pagespeed_headers =
+ ctx->base_fetch->response_headers();
+ for (int i = 0 ; i < pagespeed_headers->NumAttributes() ; i++) {
+ const GoogleString& name_gs = pagespeed_headers->Name(i);
+ const GoogleString& value_gs = pagespeed_headers->Value(i);
+
+ // We should avoid touching these fields, as ATS will drop keepalive when we do.
+ if ( StringCaseEqual(name_gs, "Connection") || StringCaseEqual(name_gs, "Transfer-Encoding") ) {
+ continue;
+ }
+
+ TSMLoc field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, name_gs.data(), name_gs.size());
+ if (field_loc != NULL) {
+ TSMimeHdrFieldValuesClear(bufp, hdr_loc, field_loc);
+ TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, field_loc, -1,
+ value_gs.data(), value_gs.size());
+ } else if (TSMimeHdrFieldCreate(bufp, hdr_loc, &field_loc) == TS_SUCCESS) {
+ if (TSMimeHdrFieldNameSet(bufp, hdr_loc, field_loc, name_gs.data(), name_gs.size()) == TS_SUCCESS) {
+ TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, field_loc, -1,
+ value_gs.data(), value_gs.size());
+ TSMimeHdrFieldAppend(bufp, hdr_loc, field_loc);
+ } else {
+ CHECK(false) << "Field name set failure";
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ } else {
+ CHECK(false) << "Field create failure";
+ }
+ }
+
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ } else {
+ DCHECK(false) << "Could not get response headers?!";
+ }
+ }
+}
+
+static void
+copy_response_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, ResponseHeaders* psol_headers) {
+ int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc);
+ TSMLoc field_loc;
+ const char *name, *value;
+ int name_len, value_len;
+ GoogleString header;
+ for (int i = 0; i < n_mime_headers; ++i) {
+ field_loc = TSMimeHdrFieldGet(bufp, hdr_loc, i);
+ if (!field_loc) {
+ TSDebug(DEBUG_TAG, "[%s] Error while obtaining header field #%d", __FUNCTION__, i);
+ continue;
+ }
+ name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len);
+ StringPiece s_name(name, name_len);
+ int n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc);
+ for (int j = 0; j < n_field_values; ++j) {
+ value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len);
+ if ( NULL == value || !value_len ) {
+ TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]",
+ __FUNCTION__, j, name_len, name);
+ } else {
+ StringPiece s_value(value, value_len);
+ psol_headers->Add(s_name, s_value);
+ //TSDebug(DEBUG_TAG, "Add response header [%.*s:%.*s]",name_len, name, value_len, value);
+ }
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ }
+}
+
+void
+copy_request_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, RequestHeaders* psol_headers) {
+ int n_mime_headers = TSMimeHdrFieldsCount(bufp, hdr_loc);
+ TSMLoc field_loc;
+ const char *name, *value;
+ int name_len, value_len;
+ GoogleString header;
+ for (int i = 0; i < n_mime_headers; ++i) {
+ field_loc = TSMimeHdrFieldGet(bufp, hdr_loc, i);
+ if (!field_loc) {
+ TSDebug(DEBUG_TAG, "[%s] Error while obtaining header field #%d", __FUNCTION__, i);
+ continue;
+ }
+ name = TSMimeHdrFieldNameGet(bufp, hdr_loc, field_loc, &name_len);
+ StringPiece s_name(name, name_len);
+ int n_field_values = TSMimeHdrFieldValuesCount(bufp, hdr_loc, field_loc);
+ for (int j = 0; j < n_field_values; ++j) {
+ value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, j, &value_len);
+ if ( NULL == value || !value_len ) {
+ TSDebug(DEBUG_TAG, "[%s] Error while getting value #%d of header [%.*s]",
+ __FUNCTION__, j, name_len, name);
+ } else {
+ StringPiece s_value(value, value_len);
+ psol_headers->Add(s_name, s_value);
+ //TSDebug(DEBUG_TAG, "Add request header [%.*s:%.*s]",name_len, name, value_len, value);
+ }
+ }
+ TSHandleMLocRelease(bufp, hdr_loc, field_loc);
+ }
+}
+
+// TODO(oschaaf): this is not sustainable when we get more
+// configuration options like this.
+bool get_override_expiry(const StringPiece& host) {
+ TSMutexLock(config_mutex);
+ AtsHostConfig* hc = config->Find(host.data(), host.size());
+ TSMutexUnlock(config_mutex);
+ return hc->override_expiry();
+}
+
+AtsRewriteOptions* get_host_options(const StringPiece& host) {
+ TSMutexLock(config_mutex);
+ AtsRewriteOptions* r = NULL;
+ AtsHostConfig* hc = config->Find(host.data(), host.size());
+ if (hc->options() != NULL) {
+ // We return a clone here to avoid having to thing about
+ // configuration reloads and outstanding options
+ r = hc->options()->Clone();
+ }
+ TSMutexUnlock(config_mutex);
+ return r;
+}
+
+std::string get_remapped_host(TSHttpTxn txn) {
+ TSMBuffer server_req_buf;
+ TSMLoc server_req_loc;
+ std::string to_host;
+ if (TSHttpTxnServerReqGet(txn, &server_req_buf, &server_req_loc) == TS_SUCCESS
+ || TSHttpTxnCachedReqGet(txn, &server_req_buf, &server_req_loc) == TS_SUCCESS) {
+ to_host = get_header(server_req_buf, server_req_loc, "Host");
+ TSHandleMLocRelease(server_req_buf, TS_NULL_MLOC, server_req_loc);
+ } else {
+ fprintf(stderr, "@@@@@@@ FAILED \n");
+ }
+ return to_host;
+}
+
+static void
+ats_transform_init(TSCont contp, TransformCtx * ctx)
+{
+ //prepare the downstream for transforming
+ TSVConn downstream_conn;
+ TSMBuffer bufp;
+ TSMLoc hdr_loc;
+ TSMBuffer reqp;
+ TSMLoc req_hdr_loc;
+ ctx->state = transform_state_output;
+
+
+ // TODO: check cleanup flow
+ if (TSHttpTxnTransformRespGet(ctx->txn, &bufp, &hdr_loc) != TS_SUCCESS) {
+ TSError("Error TSHttpTxnTransformRespGet");
+ return;
+ }
+ if (TSHttpTxnClientReqGet(ctx->txn, &reqp, &req_hdr_loc) != TS_SUCCESS) {
+ TSError("Error TSHttpTxnClientReqGet");
+ return;
+ }
+
+ AtsServerContext* server_context = ats_process_context->server_context();
+ if (server_context->IsPagespeedResource(*ctx->gurl)) {
+ CHECK(false) << "PageSpeed resource should not get here!";
+ }
+
+ downstream_conn = TSTransformOutputVConnGet(contp);
+ ctx->downstream_buffer = TSIOBufferCreate();
+ ctx->downstream_vio = TSVConnWrite(downstream_conn, contp, TSIOBufferReaderAlloc(ctx->downstream_buffer), INT64_MAX);
+
+ // TODO(oschaaf): fix host/ip(?)
+ SystemRequestContext* system_request_context =
+ new SystemRequestContext(server_context->thread_system()->NewMutex(),
+ server_context->timer(),
+ "www.foo.com",
+ 80,
+ "127.0.0.1");
+
+ ctx->base_fetch = new AtsBaseFetch(server_context, RequestContextPtr(system_request_context),
+ ctx->downstream_vio, ctx->downstream_buffer, false);
+
+
+ RewriteOptions* options = NULL;
+ RequestHeaders* request_headers = new RequestHeaders();
+ ctx->base_fetch->SetRequestHeadersTakingOwnership(request_headers);
+ copy_request_headers_to_psol(reqp, req_hdr_loc, request_headers);
+
+ TSHttpStatus status = TSHttpHdrStatusGet(bufp, hdr_loc);
+ // TODO(oschaaf): http version
+ ctx->base_fetch->response_headers()->set_status_code(status);
+ copy_response_headers_to_psol(bufp, hdr_loc, ctx->base_fetch->response_headers());
+ ctx->base_fetch->response_headers()->ComputeCaching();
+ const char* host = ctx->gurl->HostAndPort().as_string().c_str();
+ //request_headers->Lookup1(HttpAttributes::kHost);
+ if (host != NULL && strlen(host) > 0) {
+ ctx->options = get_host_options(host);
+ }
+ bool ok = ps_determine_options(server_context,
+ ctx->options,
+ request_headers,
+ ctx->base_fetch->response_headers(),
+ &options,
+ ctx->gurl);
+
+ // Take ownership of custom_options.
+ scoped_ptr<RewriteOptions> custom_options(options);
+
+ if (!ok) {
+ TSError("Failure while determining request options for psol");
+ options = server_context->global_options();
+ } else {
+ // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
+ // parameters. Keep url_string in sync with url.
+ ctx->gurl->Spec().CopyToString(ctx->url_string);
+ }
+
+ RewriteDriver* driver;
+ if (custom_options.get() == NULL) {
+ driver = server_context->NewRewriteDriver(ctx->base_fetch->request_context());
+ } else {
+ driver = server_context->NewCustomRewriteDriver(custom_options.release(), ctx->base_fetch->request_context());
+ }
+
+ driver->SetUserAgent(ctx->user_agent->c_str());
+ driver->SetRequestHeaders(*request_headers);
+
+ bool page_callback_added = false;
+ scoped_ptr<ProxyFetchPropertyCallbackCollector>
+ property_callback(
+ ProxyFetchFactory::InitiatePropertyCacheLookup(
+ false /* is resource fetch?*/,
+ *ctx->gurl,
+ server_context,
+ options,
+ ctx->base_fetch,
+ false /* requires_blink_cohort (no longer unused) */,
+ &page_callback_added));
+
+ ctx->proxy_fetch =
+ ats_process_context->proxy_fetch_factory()->CreateNewProxyFetch(
+ *(ctx->url_string), ctx->base_fetch, driver,
+ property_callback.release(),
+ NULL /* original_content_fetch */);
+
+ TSHandleMLocRelease(reqp, TS_NULL_MLOC, req_hdr_loc);
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+}
+
+static void
+ats_transform_one(TransformCtx * ctx, TSIOBufferReader upstream_reader, int amount)
+{
+ TSIOBufferBlock downstream_blkp;
+ const char *upstream_buffer;
+ int64_t upstream_length;
+
+ while (amount > 0) {
+ downstream_blkp = TSIOBufferReaderStart(upstream_reader);
+ if (!downstream_blkp) {
+ TSError("couldn't get from IOBufferBlock");
+ return;
+ }
+
+ upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, upstream_reader, &upstream_length);
+ if (!upstream_buffer) {
+ TSError("couldn't get from TSIOBufferBlockReadStart");
+ return;
+ }
+
+ if (upstream_length > amount) {
+ upstream_length = amount;
+ }
+
+ TSDebug("ats-speed", "transform!");
+ // TODO(oschaaf): use at least the message handler from the server conrtext here?
+ if (ctx->inflater == NULL) {
+ ctx->proxy_fetch->Write(StringPiece((char*)upstream_buffer, upstream_length), ats_process_context->message_handler());
+ } else {
+ char buf[net_instaweb::kStackBufferSize];
+
+ ctx->inflater->SetInput((char*)upstream_buffer, upstream_length);
+
+ while (ctx->inflater->HasUnconsumedInput()) {
+ int num_inflated_bytes = ctx->inflater->InflateBytes(
+ buf, net_instaweb::kStackBufferSize);
+ if (num_inflated_bytes < 0) {
+ TSError("Corrupted inflation");
+ } else if (num_inflated_bytes > 0) {
+ ctx->proxy_fetch->Write(StringPiece(buf, num_inflated_bytes),
+ ats_process_context->message_handler());
+ }
+ }
+ }
+ //ctx->proxy_fetch->Flush(NULL);
+ TSIOBufferReaderConsume(upstream_reader, upstream_length);
+ amount -= upstream_length;
+ }
+ // TODO(oschaaf): get the output from the base fetch, and send it downstream.
+ // This would require proper locking around the base fetch buffer
+ // We could also have a look at directly writing to the traffic server buffers
+}
+
+
+static void
+ats_transform_finish(TransformCtx * ctx)
+{
+ if (ctx->state == transform_state_output) {
+ ctx->state = transform_state_finished;
+ ctx->proxy_fetch->Done(true);
+ ctx->proxy_fetch = NULL;
+ }
+}
+
+static void
+ats_transform_do(TSCont contp)
+{
+ TSVIO upstream_vio;
+ TransformCtx *ctx;
+ int64_t upstream_todo;
+ int64_t upstream_avail;
+ int64_t downstream_bytes_written;
+
+ ctx = (TransformCtx*)TSContDataGet(contp);
+
+ if (ctx->state == transform_state_initialized) {
+ ats_transform_init(contp, ctx);
+ }
+
+ upstream_vio = TSVConnWriteVIOGet(contp);
+ downstream_bytes_written = ctx->downstream_length;
+
+ if (!TSVIOBufferGet(upstream_vio)) {
+ ats_transform_finish(ctx);
+ return;
+ }
+
+ upstream_todo = TSVIONTodoGet(upstream_vio);
+
+ if (upstream_todo > 0) {
+ upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio));
+
+ if (upstream_todo > upstream_avail) {
+ upstream_todo = upstream_avail;
+ }
+
+ if (upstream_todo > 0) {
+ ats_transform_one(ctx, TSVIOReaderGet(upstream_vio), upstream_todo);
+ TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + upstream_todo);
+ }
+ }
+
+ if (TSVIONTodoGet(upstream_vio) > 0) {
+ if (upstream_todo > 0) {
+ if (ctx->downstream_length > downstream_bytes_written) {
+ TSVIOReenable(ctx->downstream_vio);
+ }
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, upstream_vio);
+ }
+ } else {
+ ats_transform_finish(ctx);
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, upstream_vio);
+ }
+}
+
+
+static int
+ats_pagespeed_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
+{
+ if (TSVConnClosedGet(contp)) {
+ //ats_ctx_destroy((TransformCtx*)TSContDataGet(contp));
+ TSContDestroy(contp);
+ return 0;
+ } else {
+ switch (event) {
+ case TS_EVENT_ERROR:{
+ fprintf(stderr, "ats speed transform event: [%d] TS EVENT ERROR?!\n", event);
+ TSVIO upstream_vio = TSVConnWriteVIOGet(contp);
+ TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio);
+ }
+ break;
+ case TS_EVENT_VCONN_WRITE_COMPLETE:
+ TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
+ break;
+ case TS_EVENT_VCONN_WRITE_READY:
+ ats_transform_do(contp);
+ break;
+ case TS_EVENT_IMMEDIATE:
+ ats_transform_do(contp);
+ break;
+ default:
+ DCHECK(false) << "unknown event: " << event;
+ ats_transform_do(contp);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static void
+ats_pagespeed_transform_add(TSHttpTxn txnp)
+{
+ TransformCtx* ctx = get_transaction_context(txnp);
+ CHECK(ctx);
+ if (ctx->transform_added) { // Happens with a stale cache hit
+ return;
+ } else {
+ ctx->transform_added = true;
+ }
+
+ TSHttpTxnUntransformedRespCache(txnp, 1);
+ TSHttpTxnTransformedRespCache(txnp, 0);
+
+ TSVConn connp;
+
+ connp = TSTransformCreate(ats_pagespeed_transform, txnp);
+ TSContDataSet(connp, ctx);
+ TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
+}
+
+// Returns true if a server intercept was set up
+// Which means we should not attempt any further transformation
+void
+handle_read_request_header(TSHttpTxn txnp) {
+ TSMBuffer reqp = NULL;
+ TSMLoc hdr_loc = NULL;
+ char *url = NULL;
+ int url_length = -1;
+
+ TransformCtx* ctx = ats_ctx_alloc();
+ ctx->txn = txnp;
+ TSHttpTxnArgSet(txnp, TXN_INDEX_ARG, (void*) ctx);
+ TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_SET);
+
+ if (TSHttpTxnClientReqGet(txnp, &reqp, &hdr_loc) == TS_SUCCESS) {
+ url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_length);
+ if (!url || url_length <= 0) {
+ DCHECK(false) << "Could not get url!";
+ } else {
+ std::string s_url = std::string(url,url_length);
+ GoogleUrl gurl(s_url);
+
+ ctx->url_string = new GoogleString(url, url_length);
+ ctx->gurl = new GoogleUrl(*(ctx->url_string));
+ if (!ctx->gurl->IsWebValid()) {
+ TSDebug("ats-speed", "URL != WebValid(): %s", ctx->url_string->c_str());
+ } else {
+ const char * method;
+ int method_len;
+ method = TSHttpHdrMethodGet(reqp, hdr_loc, &method_len);
+ bool head_or_get = method == TS_HTTP_METHOD_GET || method == TS_HTTP_METHOD_HEAD;
+ ctx->request_method = method;
+ GoogleString user_agent = get_header(reqp, hdr_loc, "User-Agent");
+ ctx->user_agent = new GoogleString(user_agent);
+ ctx->server_context = ats_process_context->server_context();
+ if (user_agent.find(kModPagespeedSubrequestUserAgent) != user_agent.npos) {
+ ctx->mps_user_agent = true;
+ }
+ if (ats_process_context->server_context()->IsPagespeedResource(gurl)) {
+ if (head_or_get && !ctx->mps_user_agent) {
+ ctx->resource_request = true;
+ TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET);
+ }
+ } else if (ctx->gurl->PathSansQuery() == "/pagespeed_message"
+ || ctx->gurl->PathSansQuery() == "/pagespeed_statistics"
+ || ctx->gurl->PathSansQuery() == "/pagespeed_global_statistics"
+ || ctx->gurl->PathSansQuery() == "/pagespeed_console"
+ || ctx->gurl->PathSansLeaf() == "/ats_pagespeed_static/"
+ || ctx->gurl->PathSansQuery() == "/robots.txt"
+ ) {
+ ctx->resource_request = true;
+ TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET);
+ }
+ else if (StringCaseEqual(gurl.PathSansQuery() ,"/ats_pagespeed_beacon")) {
+ ctx->beacon_request = true;
+ TSHttpTxnArgSet(txnp, TXN_INDEX_OWNED_ARG, &TXN_INDEX_OWNED_ARG_UNSET);
+ hook_beacon_intercept(txnp);
+ }
+ }
+ TSfree((void*)url);
+ } // gurl->IsWebValid() == true
+ TSHandleMLocRelease(reqp, TS_NULL_MLOC, hdr_loc);
+ } else {
+ DCHECK(false) << "Could not get client request header\n";
+ }
+
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+}
+
+bool
+cache_hit(TSHttpTxn txnp) {
+ int obj_status;
+ if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
+ // TODO(oschaaf): log warning
+ return false;
+ }
+ return obj_status == TS_CACHE_LOOKUP_HIT_FRESH;
+}
+
+static int
+transform_plugin(TSCont contp, TSEvent event, void *edata)
+{
+ TSHttpTxn txn = (TSHttpTxn) edata;
+
+ CHECK(event == TS_EVENT_HTTP_READ_RESPONSE_HDR || event == TS_EVENT_HTTP_READ_CACHE_HDR
+ || event == TS_EVENT_HTTP_SEND_REQUEST_HDR || event == TS_EVENT_HTTP_READ_REQUEST_HDR
+ || event == TS_EVENT_HTTP_TXN_CLOSE || event == TS_EVENT_HTTP_SEND_RESPONSE_HDR)
+ << "Invalid transform event";
+
+ if (event != TS_EVENT_HTTP_READ_REQUEST_HDR) {
+ // Bail if an intercept is running
+ bool is_owned = TSHttpTxnArgGet(txn, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET;
+ if (!is_owned) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ }
+
+ if (event == TS_EVENT_HTTP_SEND_RESPONSE_HDR) {
+ handle_send_response_headers(txn);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ } if (event == TS_EVENT_HTTP_TXN_CLOSE) {
+ TransformCtx* ctx = get_transaction_context(txn);
+ //if (ctx != NULL && !ctx->resource_request && !ctx->beacon_request && !ctx->html_rewrite) {
+ // For intercepted requests like beacons and resource requests, we don't own the
+ // ctx here - the interceptor does.
+
+ if (ctx != NULL) {
+ bool is_owned = TSHttpTxnArgGet(txn, TXN_INDEX_OWNED_ARG) == &TXN_INDEX_OWNED_ARG_SET;
+ if (is_owned) {
+ ats_ctx_destroy(ctx);
+ }
+ }
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ } if (event == TS_EVENT_HTTP_READ_REQUEST_HDR) {
+ handle_read_request_header(txn);
+ return 0;
+ } else if (event == TS_EVENT_HTTP_SEND_REQUEST_HDR) {
+ TSMBuffer request_header_buf = NULL;
+ TSMLoc request_header_loc = NULL;
+
+ if (TSHttpTxnServerReqGet(txn, &request_header_buf, &request_header_loc) == TS_SUCCESS) {
+ hide_accept_encoding(request_header_buf, request_header_loc, "@xxAccept-Encoding");
+ // Turn off pagespeed optimization at the origin
+ set_header(request_header_buf, request_header_loc, "PageSpeed", "off");
+ TSHandleMLocRelease(request_header_buf, TS_NULL_MLOC, request_header_loc);
+ } else {
+ CHECK(false) << "Could not find server request header";
+ }
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ } else if (event == TS_EVENT_HTTP_READ_RESPONSE_HDR) {
+ TSMBuffer request_header_buf = NULL;
+ TSMLoc request_header_loc = NULL;
+
+ if (TSHttpTxnServerReqGet(txn, &request_header_buf, &request_header_loc) == TS_SUCCESS) {
+ restore_accept_encoding(request_header_buf, request_header_loc, "@xxAccept-Encoding");
+ TSHandleMLocRelease(request_header_buf, TS_NULL_MLOC, request_header_loc);
+ } else {
+ CHECK(false) << "Could not find server request header";
+ }
+ }
+
+ CHECK(event == TS_EVENT_HTTP_READ_RESPONSE_HDR || event == TS_EVENT_HTTP_READ_CACHE_HDR);
+
+ TransformCtx* ctx = get_transaction_context(txn);
+ if (ctx == NULL) {
+ // TODO(oschaaf): document how and when this happens.
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ std::string* to_host = new std::string();
+ to_host->append(get_remapped_host(ctx->txn));
+ ctx->to_host = to_host;
+ TSMBuffer response_header_buf = NULL;
+ TSMLoc response_header_loc = NULL;
+
+ // TODO(oschaaf): from configuration!
+ bool override_expiry = false;
+
+ const char* host = ctx->gurl->HostAndPort().as_string().c_str();
+ //request_headers->Lookup1(HttpAttributes::kHost);
+ if (host != NULL && strlen(host) > 0) {
+ override_expiry = get_override_expiry(host);
+ }
+
+
+ if (ctx->mps_user_agent && override_expiry) {
+ if (TSHttpTxnServerRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) {
+ // TODO => set cacheable.
+ unset_header(response_header_buf, response_header_loc, "Cache-Control");
+ unset_header(response_header_buf, response_header_loc, "Expires");
+ unset_header(response_header_buf, response_header_loc, "Age");
+ set_header(response_header_buf, response_header_loc, "Cache-Control", "public, max-age=3600");
+ TSHandleMLocRelease(response_header_buf, TS_NULL_MLOC, response_header_loc);
+ }
+ }
+ bool ok = ctx->gurl->IsWebValid() &&
+ !(ctx->resource_request || ctx->beacon_request || ctx->mps_user_agent);
+ if (!ok) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ bool have_response_header = false;
+
+ if (TSHttpTxnServerRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) {
+ have_response_header = true;
+ if (override_expiry) {
+ unset_header(response_header_buf, response_header_loc, "Cache-Control");
+ unset_header(response_header_buf, response_header_loc, "Expires");
+ unset_header(response_header_buf, response_header_loc, "Age");
+ set_header(response_header_buf, response_header_loc, "Cache-Control", "public, max-age=3600");
+ }
+ }
+ else if (TSHttpTxnCachedRespGet(txn, &response_header_buf, &response_header_loc) == TS_SUCCESS) {
+ have_response_header = true;
+ }
+ if (!have_response_header) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ if (ok) {
+ if (ctx->request_method != TS_HTTP_METHOD_GET && ctx->request_method != TS_HTTP_METHOD_HEAD
+ && ctx->request_method != TS_HTTP_METHOD_POST) {
+ ok = false;
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ }
+
+ TSHttpStatus status = TSHttpHdrStatusGet(response_header_buf, response_header_loc);
+ if (ok) {
+ if (!(status == TS_HTTP_STATUS_OK || status == TS_HTTP_STATUS_NOT_FOUND)) {
+ ok = false;
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ }
+ if (ok) {
+ StringPiece s_content_type = get_header(response_header_buf, response_header_loc, "Content-Type");
+ const net_instaweb::ContentType* content_type =
+ net_instaweb::MimeTypeToContentType(s_content_type);
+
+ if ((content_type == NULL || !content_type->IsHtmlLike())) {
+ ok = false;
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ }
+
+ if (ok) {
+ StringPiece content_encoding = get_header(response_header_buf, response_header_loc, "Content-Encoding");
+ net_instaweb::GzipInflater::InflateType inflate_type;
+ bool is_encoded = false;
+
+ if (StringCaseEqual(content_encoding, "deflate")) {
+ is_encoded = true;
+ inflate_type = GzipInflater::kDeflate;
+ } else if (StringCaseEqual(content_encoding, "gzip")) {
+ is_encoded = true;
+ inflate_type = GzipInflater::kGzip;
+ }
+
+ if (is_encoded) {
+ ctx->inflater = new GzipInflater(inflate_type);
+ ctx->inflater->Init();
+ }
+ TSDebug(DEBUG_TAG, "Will optimize [%s]", ctx->url_string->c_str());
+ ctx->html_rewrite = true;
+ set_header(response_header_buf,response_header_loc,"@gzip_nocache","0");
+ ats_pagespeed_transform_add(txn);
+ }
+
+ TSHandleMLocRelease(response_header_buf, TS_NULL_MLOC, response_header_loc);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+
+ return 0;
+}
+
+bool RegisterPlugin() {
+ TSPluginRegistrationInfo info;
+
+ info.plugin_name = (char *)"ats_pagespeed";
+ info.vendor_name = (char *)"Apache Software Foundation";
+ info.support_email = (char *)"dev@trafficserver.apache.org";
+
+ if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) {
+ TSError("Failed to register ATSSpeed");
+ return false;
+ }
+
+ return true;
+}
+
+void cleanup_process() {
+ delete ats_process_context;
+ AtsRewriteDriverFactory::Terminate();
+ AtsRewriteOptions::Terminate();
+}
+
+static void
+process_configuration()
+{
+ AtsConfig* new_config = new AtsConfig((AtsThreadSystem*)ats_process_context->server_context()->thread_system());
+ DIR *dir;
+ struct dirent *ent;
+
+ if ((dir = opendir ("/usr/local/etc/trafficserver/psol/")) != NULL) {
+ while ((ent = readdir (dir)) != NULL) {
+ size_t len = strlen(ent->d_name);
+ if (len <= 0) continue;
+ if (ent->d_name[0] == '.') continue;
+ if (ent->d_name[len-1] == '~') continue;
+ if (ent->d_name[0] == '#') continue;
+ GoogleString s("/usr/local/etc/trafficserver/psol/");
+ s.append(ent->d_name);
+ fprintf (stderr, "parse [%s]\n", s.c_str());
+ if (!new_config->Parse(s.c_str())) {
+ TSError("Error parsing %s", s.c_str());
+ }
+ }
+ closedir (dir);
+ }
+
+ AtsConfig* old_config;
+ TSMutexLock(config_mutex);
+ fprintf(stderr, "Update configuration\n");
+ old_config = config;
+ config = new_config;
+ TSMutexUnlock(config_mutex);
+ if (old_config != NULL) {
+ delete old_config;
+ }
+}
+
+static void *
+config_notification_callback(void *data)
+{
+ int BUF_MAX = 1024 * (sizeof(struct inotify_event) + 16);
+ char buf[BUF_MAX];
+ int fd,wd;
+
+ fd = inotify_init();
+
+ if (fd < 0) {
+ perror( "inotify_init" );
+ CHECK(false) << "Failed to initialize inotify";
+ }
+
+ wd = inotify_add_watch(fd, "/usr/local/etc/trafficserver/psol/", IN_MODIFY | IN_CREATE | IN_DELETE);
+
+ while (1) {
+ int len = read(fd, buf, BUF_MAX);
+ int i = 0;
+ bool do_update = false;
+ while ( i < len ) {
+ struct inotify_event *event = ( struct inotify_event * ) &buf[ i ];
+ if ( event->len ) {
+ if (!(event->mask & IN_ISDIR)) {
+ const char* name = event->name;
+ size_t name_len = strlen(event->name);
+ if (name_len > 0 && name[0] != '.' && name[0] != '#' && name[name_len-1] != '~' ) {
+ do_update = true;
+ }
+ }
+ }
+ i += ( sizeof (struct inotify_event) ) + event->len;
+ }
+ if (do_update) {
+ process_configuration();
+ }
+ }
+
+ inotify_rm_watch( fd, wd );
+ close( fd );
+
+ return NULL;
+}
+
+
+void TSPluginInit(int argc, const char *argv[]) {
+ if (RegisterPlugin() == true) {
+ if (TSHttpArgIndexReserve("ats_pagespeed", "Stores the transaction context", &TXN_INDEX_ARG) != TS_SUCCESS) {
+ CHECK(false) << "failed to reserve an argument index";
+ }
+ if (TSHttpArgIndexReserve("ats_pagespeed", "Stores the transaction context", &TXN_INDEX_OWNED_ARG) != TS_SUCCESS) {
+ CHECK(false) << "failed to reserve an argument index";
+ }
+
+ AtsRewriteOptions::Initialize();
+ AtsRewriteDriverFactory::Initialize();
+ net_instaweb::log_message_handler::Install();
+ atexit(cleanup_process);
+ ats_process_context = new AtsProcessContext();
+ process_configuration();
+ TSCont transform_contp = TSContCreate(transform_plugin, NULL);
+ TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_READ_CACHE_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_TXN_CLOSE_HOOK, transform_contp);
+ TSHttpHookAdd(TS_HTTP_SEND_RESPONSE_HDR_HOOK, transform_contp);
+
+ setup_resource_intercept();
+ CHECK(TSThreadCreate(config_notification_callback, NULL)) << "";
+ ats_process_context->message_handler()->Message(
+ kInfo, "TSPluginInit OK");
+ }
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_pagespeed.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_pagespeed.h b/plugins/experimental/ats_pagespeed/ats_pagespeed.h
new file mode 100644
index 0000000..ccf897a
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_pagespeed.h
@@ -0,0 +1,102 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#ifndef ATS_PAGESPEED_H_
+#define ATS_PAGESPEED_H_
+
+#include <string>
+
+#include <ts/ts.h>
+
+#include "net/instaweb/util/public/google_url.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+
+namespace net_instaweb {
+
+class AtsBaseFetch;
+class AtsRewriteOptions;
+class AtsServerContext;
+class GzipInflater;
+class ProxyFetch;
+class RewriteOptions;
+class RequestHeaders;
+class ResponseHeaders;
+class ServerContext;
+
+} // namespace net_instaweb
+
+enum transform_state {
+ transform_state_initialized,
+ transform_state_output,
+ transform_state_finished
+};
+
+typedef struct
+{
+ TSHttpTxn txn;
+ TSVIO downstream_vio;
+ TSIOBuffer downstream_buffer;
+ int64_t downstream_length;
+ enum transform_state state;
+
+ net_instaweb::AtsBaseFetch* base_fetch;
+ net_instaweb::ProxyFetch* proxy_fetch;
+ net_instaweb::GzipInflater* inflater;
+
+ bool write_pending;
+ bool fetch_done;
+ GoogleString* url_string;
+ bool beacon_request;
+ bool resource_request;
+ bool mps_user_agent;
+ bool transform_added;
+ net_instaweb::GoogleUrl* gurl;
+ net_instaweb::AtsServerContext* server_context;
+ GoogleString* user_agent;
+ bool html_rewrite;
+ const char* request_method;
+ int alive;
+ net_instaweb::AtsRewriteOptions* options;
+ // TODO: Use GoogleString*
+ std::string* to_host;
+} TransformCtx;
+
+TransformCtx* get_transaction_context(TSHttpTxn txnp);
+void ats_ctx_destroy(TransformCtx * ctx);
+bool cache_hit(TSHttpTxn txnp);
+
+bool ps_determine_options(net_instaweb::ServerContext* server_context,
+ // Directory-specific options, usually null. They've already been rebased off
+ // of the global options as part of the configuration process.
+ net_instaweb::RewriteOptions* directory_options,
+ net_instaweb::RequestHeaders* request_headers,
+ net_instaweb::ResponseHeaders* response_headers,
+ net_instaweb::RewriteOptions** options,
+ net_instaweb::GoogleUrl* url);
+
+void copy_request_headers_to_psol(TSMBuffer bufp, TSMLoc hdr_loc, net_instaweb::RequestHeaders* psol_headers);
+// You will own options returned by this:
+net_instaweb::AtsRewriteOptions* get_host_options(const StringPiece& host);
+
+#endif /* ATS_PAGESPEED_H_ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_process_context.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_process_context.cc b/plugins/experimental/ats_pagespeed/ats_process_context.cc
new file mode 100644
index 0000000..f3ca481
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_process_context.cc
@@ -0,0 +1,86 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#include "ats_process_context.h"
+
+#include <vector>
+
+#include "ats_rewrite_driver_factory.h"
+#include "ats_server_context.h"
+#include "ats_message_handler.h"
+#include "ats_thread_system.h"
+
+#include "net/instaweb/automatic/public/proxy_fetch.h"
+#include "net/instaweb/util/public/pthread_shared_mem.h"
+
+namespace net_instaweb {
+
+ AtsProcessContext::AtsProcessContext() : ProcessContext() {
+ AtsThreadSystem* ts = new AtsThreadSystem();
+ message_handler_.reset(new AtsMessageHandler(ts->NewMutex()));
+ driver_factory_.reset(
+ new AtsRewriteDriverFactory(
+ *this, ts, ""/*hostname, not used*/, -1/*port, not used*/));
+ server_context_ = driver_factory()->MakeAtsServerContext();
+
+ AtsRewriteOptions* root_options_ = (AtsRewriteOptions*)driver_factory_->default_options();
+ AtsRewriteOptions* server_options = root_options_->Clone();
+ AtsRewriteOptions* options = new AtsRewriteOptions(driver_factory_->thread_system());
+ server_options->Merge(*options);
+ delete options;
+
+ server_context_->global_options()->Merge(*server_options);
+ delete server_options;
+
+ message_handler_->Message(kInfo,"global default options:\r\n[%s]",driver_factory_->default_options()->OptionsToString().c_str());
+ message_handler_->Message(kInfo,"server ctx default options:\r\n[%s]",server_context_->global_options()->OptionsToString().c_str());
+ std::vector<SystemServerContext*> server_contexts;
+ server_contexts.push_back(server_context_);
+
+ //Statistics* statistics =
+ // driver_factory_->MakeGlobalSharedMemStatistics(*(SystemRewriteOptions*)server_context_->global_options());
+ GoogleString error_message;
+ int error_index = -1;
+ Statistics* global_statistics = NULL;
+ driver_factory_.get()->PostConfig(
+ server_contexts, &error_message, &error_index, &global_statistics);
+ if (error_index != -1) {
+ server_contexts[error_index]->message_handler()->Message(
+ kError, "ngx_pagespeed is enabled. %s", error_message.c_str());
+ //return NGX_ERROR;
+ CHECK(false);
+ }
+
+ AtsRewriteDriverFactory::InitStats(global_statistics);
+
+ driver_factory()->RootInit();
+ driver_factory()->ChildInit();
+
+ proxy_fetch_factory_.reset(new ProxyFetchFactory(server_context_));
+ message_handler_->Message(kInfo, "Process context constructed");
+}
+
+AtsProcessContext::~AtsProcessContext() {
+}
+
+} // namespace net_instaweb
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_process_context.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_process_context.h b/plugins/experimental/ats_pagespeed/ats_process_context.h
new file mode 100644
index 0000000..aa344b2
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_process_context.h
@@ -0,0 +1,58 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#ifndef ATS_PROCESS_CONTEXT_H_
+#define ATS_PROCESS_CONTEXT_H_
+
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/util/public/scoped_ptr.h"
+#include "net/instaweb/rewriter/public/process_context.h"
+
+namespace net_instaweb {
+
+class AtsRewriteDriverFactory;
+class ProxyFetchFactory;
+class AtsServerContext;
+
+class AtsProcessContext : ProcessContext {
+ public:
+ explicit AtsProcessContext();
+ virtual ~AtsProcessContext();
+
+ // TODO(oschaaf): const correctness
+ MessageHandler* message_handler() { return message_handler_.get(); }
+ AtsRewriteDriverFactory* driver_factory() { return driver_factory_.get(); }
+ ProxyFetchFactory* proxy_fetch_factory() { return proxy_fetch_factory_.get(); }
+ AtsServerContext* server_context() { return server_context_; }
+ private:
+ scoped_ptr<MessageHandler> message_handler_;
+ scoped_ptr<AtsRewriteDriverFactory> driver_factory_;
+ scoped_ptr<ProxyFetchFactory> proxy_fetch_factory_;
+ AtsServerContext* server_context_;
+};
+
+
+} // namespace net_instaweb
+
+#endif // ATS_PROCESS_CONTEXT_H_
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_resource_intercept.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_resource_intercept.cc b/plugins/experimental/ats_pagespeed/ats_resource_intercept.cc
new file mode 100644
index 0000000..42ece8f
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_resource_intercept.cc
@@ -0,0 +1,363 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#include <ts/ts.h>
+
+#include <stdio.h>
+
+#include "ats_resource_intercept.h"
+
+
+#include "ats_base_fetch.h"
+#include "ats_rewrite_driver_factory.h"
+#include "ats_rewrite_options.h"
+#include "ats_server_context.h"
+#include "ats_pagespeed.h"
+
+#include "net/instaweb/http/public/request_context.h"
+#include "net/instaweb/rewriter/public/resource_fetch.h"
+#include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/system/public/system_request_context.h"
+
+#include "net/instaweb/util/public/string_writer.h"
+
+
+using namespace net_instaweb;
+
+struct InterceptCtx
+{
+ TSVConn vconn;
+ TSIOBuffer req_buffer;
+ TSIOBufferReader req_reader;
+ TSIOBuffer resp_buffer;
+ TSIOBufferReader resp_reader;
+ GoogleString* response;
+ TransformCtx* request_ctx;
+ RequestHeaders* request_headers;
+
+ InterceptCtx()
+ : vconn(NULL)
+ , req_buffer(NULL)
+ , req_reader(NULL)
+ , resp_buffer(NULL)
+ , resp_reader(NULL)
+ , response( new GoogleString() )
+ , request_ctx(NULL)
+ , request_headers(NULL)
+ {
+ };
+};
+
+static void
+shutdown (TSCont cont, InterceptCtx * intercept_ctx) {
+ if (intercept_ctx->req_reader != NULL) {
+ TSIOBufferReaderFree(intercept_ctx->req_reader);
+ intercept_ctx->req_reader = NULL;
+ }
+ if (intercept_ctx->req_buffer != NULL) {
+ TSIOBufferDestroy(intercept_ctx->req_buffer);
+ intercept_ctx->req_buffer = NULL;
+ }
+ if (intercept_ctx->resp_reader != NULL) {
+ TSIOBufferReaderFree(intercept_ctx->resp_reader);
+ intercept_ctx->resp_reader = NULL;
+ }
+ if (intercept_ctx->resp_buffer != NULL) {
+ TSIOBufferDestroy(intercept_ctx->resp_buffer);
+ intercept_ctx->resp_buffer = NULL;
+ }
+ if (intercept_ctx->vconn != NULL) {
+ TSVConnShutdown(intercept_ctx->vconn, 0, 1);
+ TSVConnClose(intercept_ctx->vconn);
+ intercept_ctx->vconn = NULL;
+ }
+ if (intercept_ctx->response != NULL) {
+ delete intercept_ctx->response;
+ intercept_ctx->response = NULL;
+ }
+ // TODO(oschaaf): think the ordering of this one through.
+ if (intercept_ctx->request_ctx) {
+ ats_ctx_destroy(intercept_ctx->request_ctx);
+ intercept_ctx->request_ctx = NULL;
+ }
+ if (intercept_ctx->request_headers != NULL) {
+ delete intercept_ctx->request_headers;
+ intercept_ctx->request_headers = NULL;
+ }
+ delete intercept_ctx;
+ TSContDestroy(cont);
+}
+
+static int
+resource_intercept(TSCont cont, TSEvent event, void *edata)
+{
+ InterceptCtx *intercept_ctx = static_cast<InterceptCtx *>(TSContDataGet(cont));
+ bool shutDown = false;
+
+ // TODO(oschaaf): have a look at https://github.com/apache/trafficserver/blob/master/plugins/experimental/esi/serverIntercept.c
+ // and see if we have any edge cases we should fix.
+ switch (event) {
+ case TS_EVENT_NET_ACCEPT: {
+ intercept_ctx->vconn = static_cast<TSVConn>(edata);
+ intercept_ctx->req_buffer = TSIOBufferCreate();
+ intercept_ctx->req_reader = TSIOBufferReaderAlloc(intercept_ctx->req_buffer);
+ intercept_ctx->resp_buffer = TSIOBufferCreate();
+ intercept_ctx->resp_reader = TSIOBufferReaderAlloc(intercept_ctx->resp_buffer);
+ TSVConnRead(intercept_ctx->vconn, cont, intercept_ctx->req_buffer, 0x7fffffff);
+ } break;
+ case TS_EVENT_VCONN_READ_READY: {
+ CHECK(intercept_ctx->request_ctx->base_fetch == NULL) << "Base fetch must not be set!";
+ CHECK(intercept_ctx->request_ctx->url_string != NULL) << "Url must be set!";
+
+ TSVConnShutdown(intercept_ctx->vconn, 1, 0);
+
+ // response will already have a size for internal pages at this point.
+ // resources, however, will have to be fetched.
+ // TODO(oschaaf): this is extremely ugly.
+ if (intercept_ctx->response->size() == 0) {
+ // TODO(oschaaf): unused - must we close / clean this up?
+ TSVIO downstream_vio = TSVConnWrite(
+ intercept_ctx->vconn, cont, intercept_ctx->resp_reader, 0x7fffffff);
+
+ AtsServerContext* server_context = intercept_ctx->request_ctx->server_context;
+
+ // TODO:(oschaaf) host/port
+ SystemRequestContext* system_request_context =
+ new SystemRequestContext(server_context->thread_system()->NewMutex(),
+ server_context->timer(),
+ "www.foo.com",// TODO(oschaaf): compute these
+ 80,
+ "127.0.0.1");
+
+ intercept_ctx->request_ctx->base_fetch = new AtsBaseFetch(
+ server_context, RequestContextPtr(system_request_context),
+ downstream_vio, intercept_ctx->resp_buffer, true);
+ intercept_ctx->request_ctx->base_fetch->set_request_headers(
+ intercept_ctx->request_headers);
+
+ RewriteOptions* options = NULL;
+
+ //const char* host = intercept_ctx->request_headers->Lookup1(HttpAttributes::kHost);
+ const char* host = intercept_ctx->request_ctx->gurl->HostAndPort().as_string().c_str();
+ if (host != NULL && strlen(host) > 0) {
+ intercept_ctx->request_ctx->options = get_host_options(host);
+ }
+
+ // TODO(oschaaf): directory options should be coming from configuration!
+ bool ok = ps_determine_options(server_context,
+ intercept_ctx->request_ctx->options,
+ intercept_ctx->request_ctx->base_fetch->request_headers(),
+ intercept_ctx->request_ctx->base_fetch->response_headers(),
+ &options,
+ intercept_ctx->request_ctx->gurl);
+
+ // Take ownership of custom_options.
+ scoped_ptr<RewriteOptions> custom_options(options);
+
+ if (!ok) {
+ TSError("Failure while determining request options for psol resource");
+ // options = server_context->global_options();
+ } else {
+ // ps_determine_options modified url, removing any ModPagespeedFoo=Bar query
+ // parameters. Keep url_string in sync with url.
+ // TODO(oschaaf): we really should determine if we have to do the lookup
+ intercept_ctx->request_ctx->gurl->Spec().CopyToString(intercept_ctx->request_ctx->url_string);
+ }
+
+ // The url we have here is already checked for IsWebValid()
+ net_instaweb::ResourceFetch::Start(
+ GoogleUrl(*intercept_ctx->request_ctx->url_string),
+ custom_options.release() /* null if there aren't custom options */,
+ false /* using_spdy */, server_context, intercept_ctx->request_ctx->base_fetch);
+ } else {
+ int64_t numBytesToWrite, numBytesWritten;
+ numBytesToWrite = intercept_ctx->response->size();
+ numBytesWritten = TSIOBufferWrite(intercept_ctx->resp_buffer,
+ intercept_ctx->response->c_str(), numBytesToWrite);
+
+ if (numBytesWritten == numBytesToWrite) {
+ TSVConnWrite(intercept_ctx->vconn, cont, intercept_ctx->resp_reader, numBytesToWrite);
+ } else {
+ TSError("Not all output could be written in one go");
+ DCHECK(false);
+ }
+ }
+ } break;
+ case TS_EVENT_VCONN_EOS:
+ TSVConnShutdown(intercept_ctx->vconn, 1, 0);
+ break;
+ case TS_EVENT_VCONN_READ_COMPLETE: {
+ TSVConnShutdown(intercept_ctx->vconn, 1, 0);
+ } break;
+ case TS_EVENT_VCONN_WRITE_READY:
+ break;
+ case TS_EVENT_VCONN_WRITE_COMPLETE:
+ shutDown = true;
+ break;
+ case TS_EVENT_ERROR:
+ TSError("vconn event: error %s", intercept_ctx->request_ctx->url_string->c_str());
+ shutDown = true;
+ break;
+ case TS_EVENT_NET_ACCEPT_FAILED:
+ TSError("vconn event: accept failed");
+ shutDown = true;
+ break;
+ case TS_EVENT_IMMEDIATE:
+ case TS_EVENT_TIMEOUT:
+ break;
+ default:
+ TSError("default clause event: %d", event);
+ break;
+ }
+
+ if (shutDown) {
+ shutdown(cont, intercept_ctx);
+ }
+
+ return 1;
+}
+
+// We intercept here because serving from ats's own cache is faster
+// then serving from pagespeed's cache. (which needs to be looked in to)
+static int
+read_cache_header_callback(TSCont cont, TSEvent event, void *edata)
+{
+ TSHttpTxn txn = static_cast<TSHttpTxn>(edata);
+ TransformCtx* ctx = get_transaction_context(txn);
+
+ if (ctx == NULL) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ if (!ctx->resource_request) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+ // TODO(oschaaf): FIXME: Ownership of ctx has become too mucky.
+ // This is because I realised too late that the intercepts
+ // are able to outlive the transaction, which I hacked
+ // to work.
+ if (TSHttpIsInternalRequest(txn) == TS_SUCCESS) {
+ ats_ctx_destroy(ctx);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ if (cache_hit(txn)) {
+ ats_ctx_destroy(ctx);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ AtsServerContext* server_context = ctx->server_context;
+ AtsRewriteDriverFactory* factory = (AtsRewriteDriverFactory*)server_context->factory();
+ GoogleString output;
+ StringWriter writer(&output);
+ HttpStatus::Code status = HttpStatus::kOK;
+ ContentType content_type = kContentTypeHtml;
+ StringPiece cache_control = HttpAttributes::kNoCache;
+ const char* error_message = NULL;
+ StringPiece request_uri_path = ctx->gurl->PathAndLeaf();
+
+ if (false && ctx->gurl->PathSansQuery() == "/robots.txt") {
+ content_type = kContentTypeText;
+ writer.Write("User-agent: *\n", server_context->message_handler());
+ writer.Write("Disallow: /\n", server_context->message_handler());
+ }
+
+ // TODO(oschaaf): /pagespeed_admin handling
+ else {
+ // Optimized resource are highly cacheable (1 year expiry)
+ // TODO(oschaaf): configuration
+ TSHttpTxnRespCacheableSet(txn, 1);
+ TSHttpTxnReqCacheableSet(txn, 1);
+
+ TSMBuffer reqp;
+ TSMLoc req_hdr_loc;
+ if (TSHttpTxnClientReqGet(ctx->txn, &reqp, &req_hdr_loc) != TS_SUCCESS) {
+ TSError("Error TSHttpTxnClientReqGet for resource!");
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ TSCont interceptCont = TSContCreate(resource_intercept, TSMutexCreate());
+ InterceptCtx *intercept_ctx = new InterceptCtx();
+ intercept_ctx->request_ctx = ctx;
+ intercept_ctx->request_headers = new RequestHeaders();
+ copy_request_headers_to_psol(reqp, req_hdr_loc, intercept_ctx->request_headers);
+ TSHandleMLocRelease(reqp, TS_NULL_MLOC, req_hdr_loc);
+
+
+ TSContDataSet(interceptCont, intercept_ctx);
+ TSHttpTxnServerIntercept(interceptCont, txn);
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+ }
+
+ if (error_message != NULL) {
+ status = HttpStatus::kNotFound;
+ content_type = kContentTypeHtml;
+ output = error_message;
+ }
+
+ ResponseHeaders response_headers;
+ response_headers.SetStatusAndReason(status);
+ response_headers.set_major_version(1);
+ response_headers.set_minor_version(0);
+
+ response_headers.Add(HttpAttributes::kContentType, content_type.mime_type());
+
+ int64 now_ms = factory->timer()->NowMs();
+ response_headers.SetDate(now_ms);
+ response_headers.SetLastModified(now_ms);
+ response_headers.Add(HttpAttributes::kCacheControl, cache_control);
+
+ if (FindIgnoreCase(cache_control, "private") ==
+ static_cast<int>(StringPiece::npos)) {
+ response_headers.Add(HttpAttributes::kEtag, "W/\"0\"");
+ }
+
+ GoogleString header;
+ StringWriter header_writer(&header);
+ response_headers.WriteAsHttp(&header_writer, server_context->message_handler());
+
+ TSCont interceptCont = TSContCreate(resource_intercept, TSMutexCreate());
+ InterceptCtx *intercept_ctx = new InterceptCtx();
+ intercept_ctx->request_ctx = ctx;
+ header.append(output);
+ TSHttpTxnRespCacheableSet(txn, 0);
+ TSHttpTxnReqCacheableSet(txn, 0);
+ TSContDataSet(interceptCont, intercept_ctx);
+ TSHttpTxnServerIntercept(interceptCont, txn);
+ intercept_ctx->response->append(header);
+
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_CONTINUE);
+ return 0;
+}
+
+void setup_resource_intercept()
+{
+ TSCont cont = TSContCreate(read_cache_header_callback, NULL);
+ TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, cont);
+}
+
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_resource_intercept.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_resource_intercept.h b/plugins/experimental/ats_pagespeed/ats_resource_intercept.h
new file mode 100644
index 0000000..933f20f
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_resource_intercept.h
@@ -0,0 +1,29 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#ifndef ATS_RESOURCE_INTERCEPT_H
+#define ATS_RESOURCE_INTERCEPT_H
+
+void setup_resource_intercept();
+
+#endif // ATS_INTERCEPT_H
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.cc b/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.cc
new file mode 100644
index 0000000..4ca3d87
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.cc
@@ -0,0 +1,196 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#include "ats_rewrite_driver_factory.h"
+
+#include <cstdio>
+#include <vector>
+
+#include "ats_thread_system.h"
+#include "ats_message_handler.h"
+#include "ats_server_context.h"
+
+#include "net/instaweb/http/public/content_type.h"
+#include "net/instaweb/http/public/rate_controller.h"
+#include "net/instaweb/http/public/rate_controlling_url_async_fetcher.h"
+#include "net/instaweb/http/public/wget_url_fetcher.h"
+#include "net/instaweb/rewriter/public/rewrite_driver.h"
+#include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
+#include "net/instaweb/rewriter/public/server_context.h"
+#include "net/instaweb/rewriter/public/static_asset_manager.h"
+#include "net/instaweb/system/public/in_place_resource_recorder.h"
+#include "net/instaweb/system/public/serf_url_async_fetcher.h"
+#include "net/instaweb/system/public/system_caches.h"
+#include "net/instaweb/system/public/system_rewrite_options.h"
+#include "net/instaweb/util/public/google_message_handler.h"
+#include "net/instaweb/util/public/null_shared_mem.h"
+#include "net/instaweb/util/public/posix_timer.h"
+#include "net/instaweb/util/public/property_cache.h"
+#include "net/instaweb/util/public/pthread_shared_mem.h"
+#include "net/instaweb/util/public/scheduler_thread.h"
+#include "net/instaweb/util/public/shared_circular_buffer.h"
+#include "net/instaweb/util/public/shared_mem_statistics.h"
+#include "net/instaweb/util/public/slow_worker.h"
+#include "net/instaweb/util/public/stdio_file_system.h"
+#include "net/instaweb/util/public/string.h"
+#include "net/instaweb/util/public/string_util.h"
+#include "net/instaweb/util/public/thread_system.h"
+
+
+namespace net_instaweb {
+
+
+ AtsRewriteDriverFactory::AtsRewriteDriverFactory(
+ const ProcessContext& process_context,
+ AtsThreadSystem* thread_system,
+ StringPiece hostname, int port)
+ : SystemRewriteDriverFactory(process_context,
+ thread_system, NULL /*default shared mem runtime*/,
+ "" /*hostname, not used*/, -1/*port, not used*/)
+ , ats_message_handler_(new AtsMessageHandler(thread_system->NewMutex()))
+ , ats_html_parse_message_handler_(new AtsMessageHandler(thread_system->NewMutex()))
+ , use_per_vhost_statistics_(false)
+ , threads_started_(false)
+ {
+ InitializeDefaultOptions();
+ default_options()->set_beacon_url("/ats_pagespeed_beacon");
+ default_options()->set_enabled(RewriteOptions::kEnabledOn);
+ default_options()->SetRewriteLevel(RewriteOptions::kCoreFilters);
+
+ SystemRewriteOptions* system_options = dynamic_cast<SystemRewriteOptions*>(
+ default_options());
+ system_options->set_log_dir("/tmp/ps_log/");
+ system_options->set_statistics_logging_enabled(true);
+
+ system_options->set_file_cache_clean_inode_limit(500000);
+ system_options->set_file_cache_clean_size_kb(1024*10000);// 10 GB
+ system_options->set_avoid_renaming_introspective_javascript(true);
+ system_options->set_file_cache_path("/tmp/ats_ps/");
+ system_options->set_lru_cache_byte_limit(163840);
+ system_options->set_lru_cache_kb_per_process(1024*500);//500 MB
+
+ system_options->set_flush_html(true);
+
+ AtsRewriteOptions* ats_options = (AtsRewriteOptions*)system_options;
+ std::vector<std::string> args;
+ args.push_back("RateLimitBackgroundFetches");
+ args.push_back("on");
+ global_settings settings;
+ const char* msg = ats_options->ParseAndSetOptions(args, ats_message_handler_, settings);
+ CHECK(!msg);
+
+ set_message_buffer_size(1024*128);
+ set_message_handler(ats_message_handler_);
+ set_html_parse_message_handler(ats_html_parse_message_handler_);
+ StartThreads();
+ }
+
+ AtsRewriteDriverFactory::~AtsRewriteDriverFactory() {
+ ShutDown();
+ delete ats_message_handler_;
+ ats_message_handler_ = NULL;
+ delete ats_html_parse_message_handler_;
+ ats_html_parse_message_handler_ = NULL;
+ STLDeleteElements(&uninitialized_server_contexts_);
+ }
+
+ void AtsRewriteDriverFactory::InitStaticAssetManager(StaticAssetManager* static_js_manager) {
+ static_js_manager->set_library_url_prefix("/ats_pagespeed_static/");
+ }
+
+ Hasher* AtsRewriteDriverFactory::NewHasher() {
+ return new MD5Hasher;
+ }
+
+ MessageHandler* AtsRewriteDriverFactory::DefaultHtmlParseMessageHandler() {
+ return ats_html_parse_message_handler_;
+ }
+
+ MessageHandler* AtsRewriteDriverFactory::DefaultMessageHandler() {
+ return ats_message_handler_;
+ }
+
+ FileSystem* AtsRewriteDriverFactory::DefaultFileSystem() {
+ return new StdioFileSystem();
+ }
+
+ Timer* AtsRewriteDriverFactory::DefaultTimer() {
+ return new PosixTimer;
+ }
+
+ NamedLockManager* AtsRewriteDriverFactory::DefaultLockManager() {
+ CHECK(false) << "default lock manager should not be called";
+ return NULL;
+ }
+
+ RewriteOptions* AtsRewriteDriverFactory::NewRewriteOptions() {
+ AtsRewriteOptions* options = new AtsRewriteOptions(thread_system());
+ options->SetRewriteLevel(RewriteOptions::kCoreFilters);
+ return options;
+ }
+
+ ServerContext* AtsRewriteDriverFactory::NewDecodingServerContext() {
+ ServerContext* sc = new AtsServerContext(this);
+ InitStubDecodingServerContext(sc);
+ return sc;
+ }
+
+ void AtsRewriteDriverFactory::InitStats(Statistics* statistics) {
+ // Init standard PSOL stats.
+ SystemRewriteDriverFactory::InitStats(statistics);
+ // Init Ats-specific stats.
+ AtsServerContext::InitStats(statistics);
+ }
+
+
+ AtsServerContext* AtsRewriteDriverFactory::MakeAtsServerContext() {
+ AtsServerContext* server_context = new AtsServerContext(this);
+ uninitialized_server_contexts_.insert(server_context);
+ return server_context;
+ }
+
+ ServerContext* AtsRewriteDriverFactory::NewServerContext() {
+ LOG(DFATAL) << "MakeAtsServerContext should be used instead";
+ return NULL;
+ }
+
+net_instaweb::QueuedWorkerPool* AtsRewriteDriverFactory::CreateWorkerPool(net_instaweb::RewriteDriverFactory::WorkerPoolCategory pool,
+ StringPiece name) {
+ int tc = 8;
+ TSDebug("ats_pagespeed", "Created new QueuedWorkerPool of type %d named '%s' of size %d", pool, name.data(), tc);
+ net_instaweb::QueuedWorkerPool *q_pool = new net_instaweb::QueuedWorkerPool(tc, name, thread_system());
+ return q_pool;
+}
+
+void AtsRewriteDriverFactory::StartThreads() {
+ if (threads_started_) {
+ CHECK(false) << "threads already started";
+ }
+ SchedulerThread* thread = new SchedulerThread(thread_system(), scheduler());
+ bool ok = thread->Start();
+ CHECK(ok) << "Unable to start scheduler thread";
+ defer_cleanup(thread->MakeDeleter());
+ threads_started_ = true;
+}
+
+} // namespace net_instaweb
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.h
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.h b/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.h
new file mode 100644
index 0000000..de18a28
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_rewrite_driver_factory.h
@@ -0,0 +1,113 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#ifndef ATS_REWRITE_DRIVER_FACTORY_H_
+#define ATS_REWRITE_DRIVER_FACTORY_H_
+
+#include <set>
+
+#include "net/instaweb/system/public/system_rewrite_driver_factory.h"
+#include "net/instaweb/util/public/md5_hasher.h"
+#include "net/instaweb/util/public/scoped_ptr.h"
+
+
+namespace net_instaweb {
+
+
+ class AbstractSharedMem;
+ //class NgxMessageHandler;
+ //class NgxRewriteOptions;
+ class AtsServerContext;
+ class AtsThreadSystem;
+ class GoogleMessageHandler;
+ //class NgxUrlAsyncFetcher;
+ class SharedCircularBuffer;
+ class SharedMemRefererStatistics;
+ class SharedMemStatistics;
+ class SlowWorker;
+ class StaticAssetManager;
+ class Statistics;
+ class StaticAssetManager;
+ //class SystemCaches;
+
+class AtsRewriteDriverFactory : public SystemRewriteDriverFactory {
+ public:
+ explicit AtsRewriteDriverFactory(const ProcessContext& process_context,
+ AtsThreadSystem* thread_system,
+ StringPiece hostname, int port);
+ virtual ~AtsRewriteDriverFactory();
+
+ virtual Hasher* NewHasher();
+ virtual MessageHandler* DefaultHtmlParseMessageHandler();
+ virtual MessageHandler* DefaultMessageHandler();
+ virtual FileSystem* DefaultFileSystem();
+ virtual Timer* DefaultTimer();
+ virtual NamedLockManager* DefaultLockManager();
+ virtual RewriteOptions* NewRewriteOptions();
+ virtual ServerContext* NewDecodingServerContext();
+
+ virtual bool UseBeaconResultsInFilters() const {
+ return true;
+ }
+
+ virtual void InitStaticAssetManager(StaticAssetManager* static_js_manager);
+
+ // Initializes all the statistics objects created transitively by
+ // AtsRewriteDriverFactory, including nginx-specific and
+ // platform-independent statistics.
+ static void InitStats(Statistics* statistics);
+
+ virtual net_instaweb::QueuedWorkerPool* CreateWorkerPool(WorkerPoolCategory pool,
+ StringPiece name);
+ virtual void NonStaticInitStats(Statistics* statistics) {
+ InitStats(statistics);
+ }
+
+ AtsServerContext* MakeAtsServerContext();
+ ServerContext* NewServerContext();
+ //AbstractSharedMem* shared_mem_runtime() const {
+ // return shared_mem_runtime_.get();
+ //}
+
+ // Starts pagespeed threads if they've not been started already. Must be
+ // called after the caller has finished any forking it intends to do.
+ void StartThreads();
+ bool use_per_vhost_statistics() const {
+ return use_per_vhost_statistics_;
+ }
+ void set_use_per_vhost_statistics(bool x) {
+ use_per_vhost_statistics_ = x;
+ }
+
+ protected:
+ private:
+ //scoped_ptr<AbstractSharedMem> shared_mem_runtime_;
+ GoogleMessageHandler* ats_message_handler_;
+ GoogleMessageHandler* ats_html_parse_message_handler_;
+ bool use_per_vhost_statistics_;
+ bool threads_started_;
+};
+
+} // namespace net_instaweb
+
+#endif // ATS_REWRITE_DRIVER_FACTORY_H_
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/1fe51067/plugins/experimental/ats_pagespeed/ats_rewrite_options.cc
----------------------------------------------------------------------
diff --git a/plugins/experimental/ats_pagespeed/ats_rewrite_options.cc b/plugins/experimental/ats_pagespeed/ats_rewrite_options.cc
new file mode 100644
index 0000000..172db83
--- /dev/null
+++ b/plugins/experimental/ats_pagespeed/ats_rewrite_options.cc
@@ -0,0 +1,263 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+#include "ats_rewrite_options.h"
+
+#include "net/instaweb/public/version.h"
+#include "net/instaweb/rewriter/public/rewrite_options.h"
+#include "net/instaweb/util/public/timer.h"
+
+#include "net/instaweb/util/public/message_handler.h"
+#include "net/instaweb/rewriter/public/file_load_policy.h"
+
+#include "net/instaweb/util/public/stdio_file_system.h"
+
+#include "ats_message_handler.h"
+#include "ats_rewrite_driver_factory.h"
+
+using namespace std;
+
+namespace net_instaweb {
+
+
+RewriteOptions::Properties* AtsRewriteOptions::ats_properties_ = NULL;
+
+AtsRewriteOptions::AtsRewriteOptions(ThreadSystem* thread_system)
+ : SystemRewriteOptions(thread_system) {
+
+ Init();
+}
+
+void AtsRewriteOptions::Init() {
+ DCHECK(ats_properties_ != NULL)
+ << "Call AtsRewriteOptions::Initialize() before construction";
+ InitializeOptions(ats_properties_);
+}
+
+void AtsRewriteOptions::AddProperties() {
+ MergeSubclassProperties(ats_properties_);
+ AtsRewriteOptions dummy_config(NULL);
+
+ dummy_config.set_default_x_header_value(MOD_PAGESPEED_VERSION_STRING "-" LASTCHANGE_STRING);
+}
+
+void AtsRewriteOptions::Initialize() {
+ if (Properties::Initialize(&ats_properties_)) {
+ SystemRewriteOptions::Initialize();
+ AddProperties();
+ }
+}
+
+void AtsRewriteOptions::Terminate() {
+ if (Properties::Terminate(&ats_properties_)) {
+ SystemRewriteOptions::Terminate();
+ }
+}
+
+bool AtsRewriteOptions::IsDirective(StringPiece config_directive,
+ StringPiece compare_directive) {
+ return StringCaseEqual(config_directive, compare_directive);
+}
+
+RewriteOptions::OptionSettingResult AtsRewriteOptions::ParseAndSetOptions0(
+ StringPiece directive, GoogleString* msg, MessageHandler* handler) {
+ if (IsDirective(directive, "on")) {
+ set_enabled(RewriteOptions::kEnabledOn);
+ } else if (IsDirective(directive, "off")) {
+ set_enabled(RewriteOptions::kEnabledOff);
+ } else if (IsDirective(directive, "unplugged")) {
+ set_enabled(RewriteOptions::kEnabledUnplugged);
+ } else {
+ return RewriteOptions::kOptionNameUnknown;
+ }
+ return RewriteOptions::kOptionOk;
+}
+
+
+RewriteOptions::OptionSettingResult
+AtsRewriteOptions::ParseAndSetOptionFromName1(
+ StringPiece name, StringPiece arg,
+ GoogleString* msg, MessageHandler* handler) {
+ // FileCachePath needs error checking.
+ if (StringCaseEqual(name, kFileCachePath)) {
+ if (!StringCaseStartsWith(arg, "/")) {
+ *msg = "must start with a slash";
+ return RewriteOptions::kOptionValueInvalid;
+ }
+ }
+
+ return SystemRewriteOptions::ParseAndSetOptionFromName1(
+ name, arg, msg, handler);
+}
+
+bool AtsRewriteOptions::SetBoolFlag(bool* v, StringPiece arg) {
+ if (IsDirective(arg, "on")) {
+ *v=true;
+ return true;
+ } else if (IsDirective(arg, "off")) {
+ *v=false;
+ return true;
+ }
+ return false;
+}
+
+const char*
+AtsRewriteOptions::ParseAndSetOptions(
+ vector<string> args, MessageHandler* handler, global_settings& global_config) {
+ int n_args = args.size();
+ CHECK_GE(n_args, 1);
+
+ StringPiece directive = args[0];
+
+ // Remove initial "ModPagespeed" if there is one.
+ StringPiece mod_pagespeed("ModPagespeed");
+ if (StringCaseStartsWith(directive, mod_pagespeed)) {
+ directive.remove_prefix(mod_pagespeed.size());
+ }
+
+ GoogleString msg;
+ OptionSettingResult result;
+ if (n_args == 1) {
+ result = ParseAndSetOptions0(directive, &msg, handler);
+ } else if (n_args == 2) {
+ StringPiece arg = args[1];
+ if (IsDirective(directive, "UsePerVHostStatistics")) {
+ if (!SetBoolFlag(&global_config.use_per_vhost_statistics,arg)) {
+ msg = "Failed to set UsePerVHostStatistics value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } /* else if (IsDirective(directive, "InstallCrashHandler")) {
+ // Not applicable
+ } */ else if (IsDirective(directive, "MessageBufferSize")) {
+ int message_buffer_size;
+ bool ok = StringToInt(arg.as_string(), &message_buffer_size);
+ if (ok && message_buffer_size >= 0) {
+ global_config.message_buffer_size = message_buffer_size;
+ result = RewriteOptions::kOptionOk;
+ } else {
+ msg = "Failed to set MessageBufferSize value";
+ result = RewriteOptions::kOptionValueInvalid;
+ }
+ } else if (IsDirective(directive, "UseNativeFetcher")) {
+ if (!SetBoolFlag(&global_config.info_urls_local_only,arg)) {
+ msg = "Failed to set UseNativeFetcher value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ msg = "Native fetcher is not available in this release";
+
+ result = RewriteOptions::kOptionValueInvalid;
+ }
+ } else if (IsDirective(directive, "InfoUrlsLocalOnly")) {
+ if (!SetBoolFlag(&global_config.info_urls_local_only, arg)) {
+ msg = "Failed to set InfoUrlsLocalOnly value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ }/* else if (IsDirective(directive, "RateLimitBackgroundFetches")) {
+ if (!SetBoolFlag(&global_config.rate_limit_background_fetches, arg)) {
+ msg = "Failed to set RateLimitBackgroundFetches value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } else if (IsDirective(directive, "ForceCaching")) {
+ if (!SetBoolFlag(&global_config.force_caching, arg)) {
+ msg = "Failed to set ForceCaching value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } else if (IsDirective(directive, "ListOutstandingUrlsOnError")) {
+ if (!SetBoolFlag(&global_config.list_outstanding_urls_on_error, arg)) {
+ msg = "Failed to set ListOutstandingUrlsOnError value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } else if (IsDirective(directive, "TrackOriginalContentLength")) {
+ if (!SetBoolFlag(&global_config.track_original_content_length, arg)) {
+ msg = "Failed to set TrackOriginalContentLength value";
+ result = RewriteOptions::kOptionValueInvalid;
+ } else {
+ result = RewriteOptions::kOptionOk;
+ }
+ } */else {
+ result = ParseAndSetOptionFromName1(directive, args[1], &msg, handler);
+ }
+ } else if (n_args == 3) {
+ if (StringCaseEqual(directive, "CreateSharedMemoryMetadataCache")) {
+ int64 kb = 0;
+ if (!StringToInt64(args[2], &kb) || kb < 0) {
+ result = RewriteOptions::kOptionValueInvalid;
+ msg = "size_kb must be a positive 64-bit integer";
+ } else {
+ global_config.shm_cache_size_kb = kb;
+ result = kOptionOk;
+ //bool ok = driver_factory->caches()->CreateShmMetadataCache(
+ // args[1].as_string(), kb, &msg);
+ //result = ok ? kOptionOk : kOptionValueInvalid;
+ }
+ } else {
+ result = ParseAndSetOptionFromName2(directive, args[1], args[2],
+ &msg, handler);
+ }
+ } else if (n_args == 4) {
+ result = ParseAndSetOptionFromName3(
+ directive, args[1], args[2], args[3], &msg, handler);
+ } else {
+ return "unknown option";
+ }
+
+ if (msg.size()) {
+ handler->Message(kWarning, "Error handling config line [%s]: [%s]",
+ JoinString(args, ' ').c_str(), msg.c_str());
+ }
+
+ switch (result) {
+ case RewriteOptions::kOptionOk:
+ return NULL;
+ case RewriteOptions::kOptionNameUnknown:
+ handler->Message(kWarning, "%s", JoinString(args, ' ').c_str());
+ return "unknown option";
+ case RewriteOptions::kOptionValueInvalid: {
+ handler->Message(kWarning, "%s", JoinString(args, ' ').c_str());
+ return "Invalid value";
+ }
+ }
+
+ CHECK(false);
+ return NULL;
+}
+
+AtsRewriteOptions* AtsRewriteOptions::Clone() const {
+ AtsRewriteOptions* options = new AtsRewriteOptions(this->thread_system());
+ options->Merge(*this);
+ return options;
+}
+
+
+} // namespace net_instaweb
+