You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by zw...@apache.org on 2016/07/01 02:02:08 UTC
[trafficserver] branch master updated: TS-4395 Add new plugin,
remap_purge
This is an automated email from the ASF dual-hosted git repository.
zwoop pushed a commit to branch master
in repository https://git-dual.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new e63b12d TS-4395 Add new plugin, remap_purge
e63b12d is described below
commit e63b12d03f4fdef10129bb85152c5da615cb69cd
Author: Leif Hedstrom <zw...@apache.org>
AuthorDate: Fri Apr 29 18:25:05 2016 -0600
TS-4395 Add new plugin, remap_purge
This is similar to the existing plugin to purge based on a genID stored in a
persistent storage. The difference is that the purging is done exclusively via
a restful API, and has little (no) overhead on performance (since the
generation ID is always in memory).
Example:
map http://example.com/p1 http://p1.example.com
@plugin=remap_purge.so @pparam=--path=__secret__ @pparam=--state=example_p1
And to purge:
$ curl -s -D - -X PURGE http://example.com/p1/__secret__
HTTP/1.1 200 OK
Date: Sat, 30 Apr 2016 00:09:34 GMT
Connection: close
Server: ATS/7.0.0
Content-Length: 39
Content-Type: text/html
PURGED http://example.com/p1
---
configure.ac | 1 +
doc/admin-guide/plugins/remap_purge.en.rst | 99 ++++++
plugins/experimental/Makefile.am | 1 +
plugins/experimental/{ => remap_purge}/Makefile.am | 52 +---
plugins/experimental/remap_purge/remap_purge.c | 337 +++++++++++++++++++++
5 files changed, 442 insertions(+), 48 deletions(-)
diff --git a/configure.ac b/configure.ac
index 2e40965..802f256 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1994,6 +1994,7 @@ AC_CONFIG_FILES([
plugins/experimental/mysql_remap/Makefile
plugins/experimental/regex_revalidate/Makefile
plugins/experimental/remap_stats/Makefile
+ plugins/experimental/remap_purge/Makefile
plugins/experimental/s3_auth/Makefile
plugins/experimental/ssl_cert_loader/Makefile
plugins/experimental/sslheaders/Makefile
diff --git a/doc/admin-guide/plugins/remap_purge.en.rst b/doc/admin-guide/plugins/remap_purge.en.rst
new file mode 100644
index 0000000..bae368a
--- /dev/null
+++ b/doc/admin-guide/plugins/remap_purge.en.rst
@@ -0,0 +1,99 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+.. include:: ../../common.defs
+
+.. _admin-plugins-remap-purge:
+
+Remap Purge Plugin
+******************
+
+This remap plugin allows the administrator to easily setup remotely
+controlled ``PURGE`` for the content of an entire remap rule. The actual
+purging is instant, and has no cost on the ATS server side. The
+plugin makes it easy to accomplish this, since it's a simple REST
+API to trigger the ``PURGE`` event.
+
+
+Purpose
+=======
+
+Oftentimes, it's important, or even mission critical, to be able to purge
+a large portion of an ``Apache Traffic Server``. For example, imagine that
+you've pushed a new version of the site's HTML files, and you know it's going
+to be hours, or even days, before the content expires in the cache.
+
+Using this plugin, you can now easily set the ``Cache-Control`` very long,
+yet expire a large portion of the cache instantly. This is building upon the
+:ts:cv:`proxy.config.http.cache.generation` configuration. What the plugin
+does is to provide a REST API that makes it easy to manage this generation
+ID for one or many ATS servers.
+
+Installation
+============
+
+This plugin is still experimental, but is included with |TS| when you
+build with the experimental plugins enabled via ``configure``.
+
+Configuration
+=============
+
+This plugin only functions as a remap plugin, and is therefore
+configured in :file:`remap.config`. Be aware that the ``PURGE`` requests
+are typically restricted to ``localhost``, but see :file:`ip_allow.config`
+and :file:`remap.config` how to configure these access controls.
+
+If PURGE does not work for your setup, you can enable a relaxed configuration
+which allows GET requests to also perform the purge. This is not a recommended
+configuration, since there are likely no ACLs for this now, only the secret
+protects the content from being maliciously purged.
+
+Plugin Options
+--------------
+
+There are three configuration options for this plugin::
+
+ --secret The secret the client sends to authorize the purge
+ --header The header the client sends the secret in (optional)
+ --state-file Name of the state file where we store the GenID
+ --allow-get This also allows a simple GET to perform the purge
+
+Examples
+--------
+
+
+ map https://www.example.com http://origin.example.com \
+ @plugin=purge_remap.so @pparam=--state-file=example \
+ @pparam=--header=ATS-Purger \
+ @pparam=--secret=8BFE-656DC3564C05
+
+This setups the server to PURGE using a special header for authentication::
+
+ curl -X PURGE -H "ATS-Purger: 8BFE-656DC3564C05" https://www.example.com
+
+
+The passing of the secret as a header is option, if not specified, the
+last component of the path is used instead. Example::
+
+ map https://www.example.com/docs http://docs.example.com \
+ @plugin=purge_remap.so @pparam=--state-file=example_docs \
+ @pparam=--secret=8BFE-656DC3564C05
+
+This can now be purged with an even simpler request, but be aware that
+the secret is now likely stored in access logs as well::
+
+ curl -X PURGE https://www.example.com/docs/8BFE-656DC3564C05
diff --git a/plugins/experimental/Makefile.am b/plugins/experimental/Makefile.am
index 7832f6b..04a75da 100644
--- a/plugins/experimental/Makefile.am
+++ b/plugins/experimental/Makefile.am
@@ -41,6 +41,7 @@ SUBDIRS = \
multiplexer \
regex_revalidate \
remap_stats \
+ remap_purge \
s3_auth \
ssl_cert_loader \
sslheaders \
diff --git a/plugins/experimental/Makefile.am b/plugins/experimental/remap_purge/Makefile.am
similarity index 53%
copy from plugins/experimental/Makefile.am
copy to plugins/experimental/remap_purge/Makefile.am
index 7832f6b..57ae8cf 100644
--- a/plugins/experimental/Makefile.am
+++ b/plugins/experimental/remap_purge/Makefile.am
@@ -14,52 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-SUBDIRS = \
- acme \
- authproxy \
- background_fetch \
- balancer \
- buffer_upload \
- cache_promote \
- cache_range_requests \
- cachekey \
- collapsed_connection \
- collapsed_forwarding \
- custom_redirect \
- epic \
- escalate \
- esi \
- generator \
- geoip_acl \
- header_normalize \
- hipes \
- inliner \
- memcache \
- memcached_remap \
- metalink \
- mp4 \
- multiplexer \
- regex_revalidate \
- remap_stats \
- s3_auth \
- ssl_cert_loader \
- sslheaders \
- stale_while_revalidate \
- stream_editor \
- ts_lua \
- url_sig \
- xdebug
+include $(top_srcdir)/build/plugins.mk
-if ENABLE_CPPAPI
-if BUILD_WEBP_TRANSFORM_PLUGIN
- SUBDIRS += webp_transform
-endif
-endif
-
-if HAS_MYSQL
- SUBDIRS += mysql_remap
-endif
-
-if HAS_KYOTOCABINET
- SUBDIRS += cache_key_genid
-endif
+pkglib_LTLIBRARIES = remap_purge.la
+remap_purge_la_SOURCES = remap_purge.c
+remap_purge_la_LDFLAGS = $(TS_PLUGIN_LDFLAGS)
diff --git a/plugins/experimental/remap_purge/remap_purge.c b/plugins/experimental/remap_purge/remap_purge.c
new file mode 100644
index 0000000..b21cf25
--- /dev/null
+++ b/plugins/experimental/remap_purge/remap_purge.c
@@ -0,0 +1,337 @@
+/** @file
+
+ Per-remap purge RESTful API for stateful generation ID management.
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <getopt.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "ts/ts.h"
+#include "ts/remap.h"
+#include "ts/ink_defs.h"
+
+static const char *PLUGIN_NAME = "remap_purge";
+static const char *DEFAULT_DIR = "var/trafficserver"; /* Not perfect, but no better API) */
+
+typedef struct PurgeInstance_t {
+ char *id;
+ char *secret;
+ int secret_len;
+ char *header;
+ int header_len;
+ char *state_file;
+ bool allow_get;
+ int64_t gen_id;
+ TSMutex lock;
+} PurgeInstance;
+
+static char *
+make_state_path(const char *filename)
+{
+ if ('/' == *filename) {
+ return TSstrdup(filename);
+ } else {
+ char buf[8192];
+ struct stat s;
+ const char *dir = TSInstallDirGet();
+
+ snprintf(buf, sizeof(buf), "%s/%s/%s", dir, DEFAULT_DIR, PLUGIN_NAME);
+ if (-1 == stat(buf, &s)) {
+ if (ENOENT == errno) {
+ if (-1 == mkdir(buf, S_IRWXU)) {
+ TSError("[%s] Unable to create directory %s: %s (%d)", PLUGIN_NAME, buf, strerror(errno), errno);
+ return NULL;
+ }
+ } else {
+ TSError("[%s] Unable to stat() directory %s: %s (%d)", PLUGIN_NAME, buf, strerror(errno), errno);
+ return NULL;
+ }
+ } else {
+ if (!S_ISDIR(s.st_mode)) {
+ TSError("[%s] Can not create directory %s, file exists", PLUGIN_NAME, buf);
+ return NULL;
+ }
+ }
+ snprintf(buf, sizeof(buf), "%s/%s/%s/%s.genid", dir, DEFAULT_DIR, PLUGIN_NAME, filename);
+ return TSstrdup(buf);
+ }
+
+ return NULL;
+}
+
+/* Constructor and destructor for the PurgeInstance */
+static void
+init_purge_instance(PurgeInstance *purge, char *id)
+{
+ FILE *file = fopen(purge->state_file, "r");
+
+ if (file) {
+ fscanf(file, "%" PRId64 "", &purge->gen_id);
+ TSDebug(PLUGIN_NAME, "Read genID from %s for %s", purge->state_file, purge->id);
+ fclose(file);
+ }
+
+ /* If not specified, we set the ID tag to the fromURL from the remap rule that triggered */
+ if (!purge->id) {
+ purge->id = TSstrdup(id);
+ }
+
+ purge->lock = TSMutexCreate();
+}
+
+static void
+delete_purge_instance(PurgeInstance *purge)
+{
+ if (purge) {
+ TSfree(purge->id);
+ TSfree(purge->state_file);
+ TSfree(purge->secret);
+ TSfree(purge->header);
+ TSMutexDestroy(purge->lock);
+ TSfree(purge);
+ }
+}
+
+/* This is where we start the PURGE events, setting up the transactino to fail,
+ and bump the generation ID, and finally save the state. */
+static int
+on_http_cache_lookup_complete(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge)
+{
+ FILE *file;
+
+ TSMutexLock(purge->lock);
+
+ ++purge->gen_id;
+ TSDebug(PLUGIN_NAME, "Bumping the Generation ID to %" PRId64 " for %s", purge->gen_id, purge->id);
+
+ if ((file = fopen(purge->state_file, "w"))) {
+ TSDebug(PLUGIN_NAME, "\tsaving state to %s", purge->state_file);
+ fprintf(file, "%" PRId64 "", purge->gen_id);
+ fclose(file);
+ } else {
+ TSError("[%s] Unable to save state to file %s: errno=%d", PLUGIN_NAME, purge->state_file, errno);
+ }
+
+ TSMutexUnlock(purge->lock);
+
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_ERROR);
+ return TS_SUCCESS;
+}
+
+/* Before we can send the response, we want to modify it to a "200 OK" again,
+ and produce some reasonable body output. */
+static int
+on_send_response_header(TSHttpTxn txnp, TSCont contp, PurgeInstance *purge)
+{
+ TSMBuffer bufp;
+ TSMLoc hdr_loc;
+
+ TSDebug(PLUGIN_NAME, "Fixing up the response on the successful PURGE");
+ if (TS_SUCCESS == TSHttpTxnClientRespGet(txnp, &bufp, &hdr_loc)) {
+ char response[1024];
+ int len = snprintf(response, sizeof(response), "PURGED %s\r\n\r\n", purge->id);
+
+ TSHttpHdrStatusSet(bufp, hdr_loc, TS_HTTP_STATUS_OK);
+ TSHttpHdrReasonSet(bufp, hdr_loc, "OK", 2);
+ TSHttpTxnErrorBodySet(txnp, TSstrdup(response), len >= sizeof(response) ? sizeof(response) - 1 : len, NULL);
+
+ TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
+ } else {
+ TSHttpTxnReenable(txnp, TS_EVENT_HTTP_ERROR);
+ }
+
+ return TS_SUCCESS;
+}
+
+/* This is the main continuation, triggered after DoRemap has decided we should
+ handle this request internally. */
+static int
+purge_cont(TSCont contp, TSEvent event, void *edata)
+{
+ TSHttpTxn txnp = (TSHttpTxn)edata;
+ PurgeInstance *purge = (PurgeInstance *)TSContDataGet(contp);
+
+ switch (event) {
+ case TS_EVENT_HTTP_SEND_RESPONSE_HDR:
+ return on_send_response_header(txnp, contp, purge);
+ break;
+
+ case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE:
+ return on_http_cache_lookup_complete(txnp, contp, purge);
+ break;
+
+ default:
+ TSDebug(PLUGIN_NAME, "Unexpected event: %d", event);
+ break;
+ }
+
+ return TS_SUCCESS;
+}
+
+static void
+handle_purge(TSHttpTxn txnp, PurgeInstance *purge)
+{
+ TSMBuffer reqp;
+ TSMLoc hdr_loc = NULL, url_loc = NULL;
+ bool should_purge = false;
+
+ if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &reqp, &hdr_loc)) {
+ int method_len = 0;
+ const char *method = TSHttpHdrMethodGet(reqp, hdr_loc, &method_len);
+
+ if ((TS_HTTP_METHOD_PURGE == method) || ((TS_HTTP_METHOD_GET == method) && purge->allow_get)) {
+ /* First see if we require the "secret" to be passed in a header, and then use that */
+ if (purge->header) {
+ TSMLoc field_loc = TSMimeHdrFieldFind(reqp, hdr_loc, purge->header, purge->header_len);
+
+ if (field_loc) {
+ const char *header;
+ int header_len;
+
+ header = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field_loc, -1, &header_len);
+ TSDebug(PLUGIN_NAME, "Checking for %.*s == %s ?", header_len, header, purge->secret);
+ if (header && (header_len == purge->secret_len) && !memcmp(header, purge->secret, header_len)) {
+ should_purge = true;
+ }
+ TSHandleMLocRelease(reqp, hdr_loc, field_loc);
+ }
+ } else {
+ /* We are matching on the path component instead of a header */
+ if (TS_SUCCESS == TSHttpHdrUrlGet(reqp, hdr_loc, &url_loc)) {
+ int path_len = 0;
+ const char *path = TSUrlPathGet(reqp, url_loc, &path_len);
+
+ TSDebug(PLUGIN_NAME, "Checking PATH = %.*s", path_len, path);
+ if (path && (path_len >= purge->secret_len)) {
+ const char *s_path = (const char *)memrchr(path, '/', path_len);
+
+ if (!memcmp(s_path ? s_path + 1 : path, purge->secret, purge->secret_len)) {
+ should_purge = true;
+ }
+ }
+ TSHandleMLocRelease(reqp, hdr_loc, url_loc);
+ }
+ }
+ }
+ TSHandleMLocRelease(reqp, TS_NULL_MLOC, hdr_loc);
+ }
+
+ /* Setup the continuation to handle this request if appropriate, if not, set the GenID if needed */
+ if (should_purge) {
+ TSCont cont = TSContCreate(purge_cont, TSMutexCreate());
+
+ TSContDataSet(cont, purge);
+ TSHttpTxnHookAdd(txnp, TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, cont);
+ TSHttpTxnHookAdd(txnp, TS_HTTP_SEND_RESPONSE_HDR_HOOK, cont);
+ } else if (purge->gen_id > 0) {
+ TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_CACHE_GENERATION, purge->gen_id);
+ }
+}
+
+TSReturnCode
+TSRemapInit(TSRemapInterface *api_info, char *errbuf, int errbuf_size)
+{
+ TSDebug(PLUGIN_NAME, "initialized");
+ return TS_SUCCESS;
+}
+
+TSReturnCode
+TSRemapNewInstance(int argc, char *argv[], void **ih, char *errbuf, int errbuf_size)
+{
+ char *id = argv[0]; /* The ID is default to the "from" URL, so save it */
+ PurgeInstance *purge = TSmalloc(sizeof(PurgeInstance));
+ static const struct option longopt[] = {
+ {(char *)"id", required_argument, NULL, 'i'}, {(char *)"secret", required_argument, NULL, 's'},
+ {(char *)"header", required_argument, NULL, 'h'}, {(char *)"state-file", required_argument, NULL, 'f'},
+ {(char *)"allow-get", no_argument, NULL, 'a'}, {NULL, no_argument, NULL, '\0'},
+ };
+
+ memset(purge, 0, sizeof(PurgeInstance));
+
+ // The first two arguments are the "from" and "to" URL string. We need to
+ // skip them, but we also require that there be an option to masquerade as
+ // argv[0], so we increment the argument indexes by 1 rather than by 2.
+ argc--;
+ argv++;
+
+ for (;;) {
+ int opt = getopt_long(argc, (char *const *)argv, "", longopt, NULL);
+
+ if (opt == -1) {
+ break;
+ }
+
+ switch (opt) {
+ case 'a':
+ purge->allow_get = true;
+ break;
+ case 'h':
+ purge->header = TSstrdup(optarg);
+ purge->header_len = strlen(purge->header);
+ break;
+ case 'i':
+ purge->id = TSstrdup(optarg);
+ break;
+ case 's':
+ purge->secret = TSstrdup(optarg);
+ purge->secret_len = strlen(purge->secret);
+ break;
+ case 'f':
+ purge->state_file = make_state_path(optarg);
+ break;
+ }
+ }
+
+ if ((NULL == purge->secret) || (NULL == purge->state_file) || !purge->secret_len) {
+ TSError("[%s] Unable to create remap instance, need at least a secret (--secret) and state (--state_file)", PLUGIN_NAME);
+ return TS_ERROR;
+ } else {
+ init_purge_instance(purge, id);
+ *ih = (void *)purge;
+ return TS_SUCCESS;
+ }
+}
+
+void
+TSRemapDeleteInstance(void *ih)
+{
+ PurgeInstance *purge = (PurgeInstance *)ih;
+
+ delete_purge_instance(purge);
+}
+
+TSRemapStatus
+TSRemapDoRemap(void *ih, TSHttpTxn txnp, TSRemapRequestInfo *rri)
+{
+ PurgeInstance *purge = (PurgeInstance *)ih;
+
+ handle_purge(txnp, purge);
+ return TSREMAP_NO_REMAP; // This plugin never rewrites anything.
+}
--
To stop receiving notification emails like this one, please contact
['"commits@trafficserver.apache.org" <co...@trafficserver.apache.org>'].