You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by bc...@apache.org on 2019/04/18 22:27:41 UTC
[trafficserver] branch master updated: -H "xdebug: probe" injects
trace of headers into response body
This is an automated email from the ASF dual-hosted git repository.
bcall pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficserver.git
The following commit(s) were added to refs/heads/master by this push:
new 13425d2 -H "xdebug: probe" injects trace of headers into response body
13425d2 is described below
commit 13425d228263c4cba295059a90b5888b892dc057
Author: Aaron Canary <ac...@oath.com>
AuthorDate: Wed Apr 17 13:40:31 2019 -0500
-H "xdebug: probe" injects trace of headers into response body
With the intent to debug CDN, peered or multi-tiered proxy behavior, this header will trigger the code to write the request and response headers to the response body at each proxy the request is processed.
This has been proven to be significantly easier to use than log-headers.
As with all xdebug features some security layer should be applied to control who is allowed to see this type of response data.
docs
asf license
include unistd
xdebug probe in json
using std::string_view
---
doc/admin-guide/plugins/xdebug.en.rst | 19 ++--
plugins/xdebug/xdebug.cc | 130 +++++++++++----------------
plugins/xdebug/xdebug_headers.cc | 159 ++++++++++++++++++++++++++++++++++
plugins/xdebug/xdebug_transforms.cc | 158 +++++++++++++++++++++++++++++++++
4 files changed, 382 insertions(+), 84 deletions(-)
diff --git a/doc/admin-guide/plugins/xdebug.en.rst b/doc/admin-guide/plugins/xdebug.en.rst
index 34f77a8..a9eeac7 100644
--- a/doc/admin-guide/plugins/xdebug.en.rst
+++ b/doc/admin-guide/plugins/xdebug.en.rst
@@ -57,11 +57,20 @@ Diags
transaction specific diagnostics for the transaction. This also requires
that :ts:cv:`proxy.config.diags.debug.enabled` is set to ``1``.
-log-headers
- If the ``log-headers`` is requested while :ts:cv:`proxy.config.diags.debug.tags`
- is set to ``xdebug.headers`` and :ts:cv:`proxy.config.diags.debug.enabled` is set to ``1``,
- then all client and server, request and response headers are logged.
- Also, the ``X-Debug: log-headers`` header is always added to the upstream request.
+Probe
+ All request and response headers are written to the response body. Because
+ the body is altered, it disables writing to cache.
+ In conjuction with the `fwd` tag, the response body will contain a
+ chronological log of all headers for all transactions used for this
+ response.
+
+ Layout:
+
+ - Request Headers from Client -> Proxy A
+ - Request Headers from Proxy A -> Proxy B
+ - Original content body
+ - Response Headers from Proxy B -> Proxy A
+ - Response Headers from Proxy A -> Client
X-Cache-Key
The ``X-Cache-Key`` header contains the URL that identifies the HTTP object in the
diff --git a/plugins/xdebug/xdebug.cc b/plugins/xdebug/xdebug.cc
index c6eefd0..a2d54b8 100644
--- a/plugins/xdebug/xdebug.cc
+++ b/plugins/xdebug/xdebug.cc
@@ -26,13 +26,15 @@
#include <cstdint>
#include <cinttypes>
#include <string_view>
+#include <unistd.h>
#include <ts/ts.h>
#include "tscore/ink_defs.h"
#include "tscpp/util/PostScript.h"
#include "tscpp/util/TextView.h"
-#define DEBUG_TAG_LOG_HEADERS "xdebug.headers"
+#include "xdebug_headers.cc"
+#include "xdebug_transforms.cc"
static struct {
const char *str;
@@ -47,9 +49,11 @@ enum {
XHEADER_X_TRANSACTION_ID = 1u << 6,
XHEADER_X_DUMP_HEADERS = 1u << 7,
XHEADER_X_REMAP = 1u << 8,
+ XHEADER_X_PROBE_HEADERS = 1u << 9,
};
static int XArgIndex = 0;
+static int BodyBuilderArgIndex = 0;
static TSCont XInjectHeadersCont = nullptr;
static TSCont XDeleteDebugHdrCont = nullptr;
@@ -321,48 +325,6 @@ InjectTxnUuidHeader(TSHttpTxn txn, TSMBuffer buffer, TSMLoc hdr)
}
}
-///////////////////////////////////////////////////////////////////////////
-// Dump a header on stderr, useful together with TSDebug().
-void
-log_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, const char *msg_type)
-{
- if (!TSIsDebugTagSet(DEBUG_TAG_LOG_HEADERS)) {
- return;
- }
-
- TSIOBuffer output_buffer;
- TSIOBufferReader reader;
- TSIOBufferBlock block;
- const char *block_start;
- int64_t block_avail;
-
- std::stringstream ss;
- ss << "TxnID:" << TSHttpTxnIdGet(txn) << " " << msg_type << " Headers are...";
-
- output_buffer = TSIOBufferCreate();
- reader = TSIOBufferReaderAlloc(output_buffer);
-
- /* This will print just MIMEFields and not the http request line */
- TSMimeHdrPrint(bufp, hdr_loc, output_buffer);
-
- /* We need to loop over all the buffer blocks, there can be more than 1 */
- block = TSIOBufferReaderStart(reader);
- do {
- block_start = TSIOBufferBlockReadStart(block, reader, &block_avail);
- if (block_avail > 0) {
- ss << "\n" << std::string(block_start, static_cast<int>(block_avail));
- }
- TSIOBufferReaderConsume(reader, block_avail);
- block = TSIOBufferReaderStart(reader);
- } while (block && block_avail != 0);
-
- /* Free up the TSIOBuffer that we used to print out the header */
- TSIOBufferReaderFree(reader);
- TSIOBufferDestroy(output_buffer);
-
- TSDebug(DEBUG_TAG_LOG_HEADERS, "%s", ss.str().c_str());
-}
-
static int
XInjectResponseHeaders(TSCont /* contp */, TSEvent event, void *edata)
{
@@ -401,12 +363,25 @@ XInjectResponseHeaders(TSCont /* contp */, TSEvent event, void *edata)
InjectTxnUuidHeader(txn, buffer, hdr);
}
+ if (xheaders & XHEADER_X_REMAP) {
+ InjectRemapHeader(txn, buffer, hdr);
+ }
+
+ // intentionally placed after all injected headers.
+
if (xheaders & XHEADER_X_DUMP_HEADERS) {
log_headers(txn, buffer, hdr, "ClientResponse");
}
- if (xheaders & XHEADER_X_REMAP) {
- InjectRemapHeader(txn, buffer, hdr);
+ if (xheaders & XHEADER_X_PROBE_HEADERS) {
+ BodyBuilder *data = static_cast<BodyBuilder *>(TSHttpTxnArgGet(txn, BodyBuilderArgIndex));
+ TSDebug("xdebug_transform", "XInjectResponseHeaders(): client resp header ready");
+ if (data == nullptr) {
+ TSHttpTxnReenable(txn, TS_EVENT_HTTP_ERROR);
+ return TS_ERROR;
+ }
+ data->hdr_ready = true;
+ writePostBody(data);
}
done:
@@ -519,42 +494,36 @@ XScanRequestHeaders(TSCont /* contp */, TSEvent event, void *edata)
} else if (header_field_eq("diags", value, vsize)) {
// Enable diagnostics for DebugTxn()'s only
TSHttpTxnDebugSet(txn, 1);
- } else if (header_field_eq("log-headers", value, vsize)) {
- xheaders |= XHEADER_X_DUMP_HEADERS;
- log_headers(txn, buffer, hdr, "ClientRequest");
-
- // dump on server request
- auto send_req_dump = [](TSCont /* contp */, TSEvent event, void *edata) -> int {
- TSHttpTxn txn = (TSHttpTxn)edata;
- TSMBuffer buffer;
- TSMLoc hdr;
- if (TSHttpTxnServerReqGet(txn, &buffer, &hdr) == TS_SUCCESS) {
- // re-add header "X-Debug: log-headers", but only once
- TSMLoc dst = TSMimeHdrFieldFind(buffer, hdr, xDebugHeader.str, xDebugHeader.len);
- if (dst == TS_NULL_MLOC) {
- if (TSMimeHdrFieldCreateNamed(buffer, hdr, xDebugHeader.str, xDebugHeader.len, &dst) == TS_SUCCESS) {
- TSReleaseAssert(TSMimeHdrFieldAppend(buffer, hdr, dst) == TS_SUCCESS);
- TSReleaseAssert(TSMimeHdrFieldValueStringInsert(buffer, hdr, dst, 0 /* idx */, "log-headers",
- lengthof("log-headers")) == TS_SUCCESS);
- log_headers(txn, buffer, hdr, "ServerRequest");
- }
- }
- }
- return TS_EVENT_NONE;
- };
- TSHttpTxnHookAdd(txn, TS_HTTP_SEND_REQUEST_HDR_HOOK, TSContCreate(send_req_dump, nullptr));
-
- // dump on server response
- auto read_resp_dump = [](TSCont /* contp */, TSEvent event, void *edata) -> int {
- TSHttpTxn txn = (TSHttpTxn)edata;
- TSMBuffer buffer;
- TSMLoc hdr;
- if (TSHttpTxnServerRespGet(txn, &buffer, &hdr) == TS_SUCCESS) {
- log_headers(txn, buffer, hdr, "ServerResponse");
- }
+
+ } else if (header_field_eq("probe", value, vsize)) {
+ xheaders |= XHEADER_X_PROBE_HEADERS;
+
+ // prefix request headers and postfix response headers
+ BodyBuilder *data = new BodyBuilder();
+ data->txn = txn;
+
+ TSVConn connp = TSTransformCreate(body_transform, txn);
+ TSContDataSet(connp, data);
+ TSHttpTxnHookAdd(txn, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
+
+ // store data pointer in txnarg to use in global cont XInjectResponseHeaders
+ TSHttpTxnArgSet(txn, BodyBuilderArgIndex, data);
+
+ // create a self-cleanup on close
+ auto cleanupBodyBuilder = [](TSCont /* contp */, TSEvent event, void *edata) -> int {
+ TSHttpTxn txn = (TSHttpTxn)edata;
+ BodyBuilder *data = static_cast<BodyBuilder *>(TSHttpTxnArgGet(txn, BodyBuilderArgIndex));
+ delete data;
return TS_EVENT_NONE;
};
- TSHttpTxnHookAdd(txn, TS_HTTP_READ_RESPONSE_HDR_HOOK, TSContCreate(read_resp_dump, nullptr));
+ TSHttpTxnHookAdd(txn, TS_HTTP_TXN_CLOSE_HOOK, TSContCreate(cleanupBodyBuilder, nullptr));
+
+ // disable writing to cache because we are injecting data into the body.
+ TSHttpTxnReqCacheableSet(txn, 0);
+ TSHttpTxnRespCacheableSet(txn, 0);
+ TSHttpTxnServerRespNoStoreSet(txn, 1);
+ TSHttpTxnTransformedRespCache(txn, 0);
+ TSHttpTxnUntransformedRespCache(txn, 0);
} else if (isFwdFieldValue(std::string_view(value, vsize), fwdCnt)) {
if (fwdCnt > 0) {
@@ -666,7 +635,10 @@ TSPluginInit(int argc, const char *argv[])
// Setup the global hook
TSReleaseAssert(TSHttpTxnArgIndexReserve("xdebug", "xdebug header requests", &XArgIndex) == TS_SUCCESS);
+ TSReleaseAssert(TSHttpTxnArgIndexReserve("bodyTransform", "BodyBuilder*", &XArgIndex) == TS_SUCCESS);
TSReleaseAssert(XInjectHeadersCont = TSContCreate(XInjectResponseHeaders, nullptr));
TSReleaseAssert(XDeleteDebugHdrCont = TSContCreate(XDeleteDebugHdr, nullptr));
TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(XScanRequestHeaders, nullptr));
+
+ gethostname(Hostname, 1024);
}
diff --git a/plugins/xdebug/xdebug_headers.cc b/plugins/xdebug/xdebug_headers.cc
new file mode 100644
index 0000000..51f5d86
--- /dev/null
+++ b/plugins/xdebug/xdebug_headers.cc
@@ -0,0 +1,159 @@
+/** @file
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include <cstdlib>
+#include <stdio.h>
+#include <cstdio>
+#include <strings.h>
+#include <string_view>
+#include <sstream>
+#include <cstring>
+#include <getopt.h>
+
+#define DEBUG_TAG_LOG_HEADERS "xdebug.headers"
+
+std::string_view
+escape_char_for_json(char const &c, bool &parsing_key)
+{
+ switch (c) {
+ case '\'':
+ return {"\\\'"};
+ case '"':
+ return {"\\\""};
+ case '\\':
+ return {"\\\\"};
+ case '\b':
+ return {"\\b"};
+ case '\f':
+ return {"\\f"};
+ case '\t':
+ return {"\\t"};
+
+ // Special header reformating
+ case '\r':
+ return {""};
+ case '\n':
+ parsing_key = true;
+ return {"',\r\n\t'"}; // replace new line with pair delemiter
+ case ':':
+ if (parsing_key) {
+ return {"' : "}; // replace colon after keywith quote + colon
+ }
+ return {":"};
+ case ' ':
+ if (parsing_key) {
+ parsing_key = false;
+ return {"'"}; // replace first space after the key to be a quote
+ }
+ return {" "};
+ default:
+ return {&c, 1};
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////
+// Dump a header on stderr, useful together with TSDebug().
+void
+print_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, std::stringstream &ss)
+{
+ TSIOBuffer output_buffer;
+ TSIOBufferReader reader;
+ TSIOBufferBlock block;
+ const char *block_start;
+ int64_t block_avail;
+ bool parsing_key = true;
+ size_t print_rewind = ss.str().length();
+ output_buffer = TSIOBufferCreate();
+ reader = TSIOBufferReaderAlloc(output_buffer);
+
+ ss << "\t'";
+ /* This will print just MIMEFields and not the http request line */
+ TSMimeHdrPrint(bufp, hdr_loc, output_buffer);
+
+ /* We need to loop over all the buffer blocks, there can be more than 1 */
+ block = TSIOBufferReaderStart(reader);
+ do {
+ block_start = TSIOBufferBlockReadStart(block, reader, &block_avail);
+ for (const char *c = block_start; c < block_start + block_avail; ++c) {
+ bool was_parsing_key = parsing_key;
+ ss << escape_char_for_json(*c, parsing_key);
+ if (parsing_key && !was_parsing_key) {
+ print_rewind = ss.str().length() - 1;
+ }
+ }
+ TSIOBufferReaderConsume(reader, block_avail);
+ block = TSIOBufferReaderStart(reader);
+ } while (block && block_avail != 0);
+
+ ss.seekp(print_rewind);
+
+ /* Free up the TSIOBuffer that we used to print out the header */
+ TSIOBufferReaderFree(reader);
+ TSIOBufferDestroy(output_buffer);
+
+ TSDebug(DEBUG_TAG_LOG_HEADERS, "%s", ss.str().c_str());
+}
+
+void
+log_headers(TSHttpTxn txn, TSMBuffer bufp, TSMLoc hdr_loc, char const *type_msg)
+{
+ if (TSIsDebugTagSet(DEBUG_TAG_LOG_HEADERS)) {
+ std::stringstream output;
+ print_headers(txn, bufp, hdr_loc, output);
+ TSDebug(DEBUG_TAG_LOG_HEADERS, "\n=============\n %s headers are... \n %s", type_msg, output.str().c_str());
+ }
+}
+
+void
+print_request_headers(TSHttpTxn txn, std::stringstream &output)
+{
+ TSMBuffer buf_c, buf_s;
+ TSMLoc hdr_loc;
+ if (TSHttpTxnClientReqGet(txn, &buf_c, &hdr_loc) == TS_SUCCESS) {
+ output << "{'type':'request', 'side':'client', 'headers': {\n";
+ print_headers(txn, buf_c, hdr_loc, output);
+ output << "}}";
+ TSHandleMLocRelease(buf_c, TS_NULL_MLOC, hdr_loc);
+ }
+ if (TSHttpTxnServerReqGet(txn, &buf_s, &hdr_loc) == TS_SUCCESS) {
+ output << ",{'type':'request', 'side':'server', 'headers': {\n";
+ print_headers(txn, buf_s, hdr_loc, output);
+ output << "}}";
+ TSHandleMLocRelease(buf_s, TS_NULL_MLOC, hdr_loc);
+ }
+}
+
+void
+print_response_headers(TSHttpTxn txn, std::stringstream &output)
+{
+ TSMBuffer buf_c, buf_s;
+ TSMLoc hdr_loc;
+ if (TSHttpTxnServerRespGet(txn, &buf_s, &hdr_loc) == TS_SUCCESS) {
+ output << "{'type':'response', 'side':'server', 'headers': {\n";
+ print_headers(txn, buf_s, hdr_loc, output);
+ output << "}},";
+ TSHandleMLocRelease(buf_s, TS_NULL_MLOC, hdr_loc);
+ }
+ if (TSHttpTxnClientRespGet(txn, &buf_c, &hdr_loc) == TS_SUCCESS) {
+ output << "{'type':'response', 'side':'client', 'headers': {\n";
+ print_headers(txn, buf_c, hdr_loc, output);
+ output << "}}";
+ TSHandleMLocRelease(buf_c, TS_NULL_MLOC, hdr_loc);
+ }
+}
diff --git a/plugins/xdebug/xdebug_transforms.cc b/plugins/xdebug/xdebug_transforms.cc
new file mode 100644
index 0000000..4ceeea1
--- /dev/null
+++ b/plugins/xdebug/xdebug_transforms.cc
@@ -0,0 +1,158 @@
+/** @file
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <functional>
+#include <atomic>
+
+#include "ts/ts.h"
+
+static const std::string_view MultipartBoundary{"\r\n--- ATS xDebug Probe Injection Boundary ---\r\n\r\n"};
+
+struct BodyBuilder {
+ TSVIO output_vio = nullptr;
+ TSIOBuffer output_buffer = nullptr;
+ TSIOBufferReader output_reader = nullptr;
+ bool wrote_prebody = false;
+ bool wrote_body = false;
+ bool hdr_ready = false;
+ std::atomic_flag wrote_postbody;
+
+ int64_t nbytes = 0;
+ TSHttpTxn txn = nullptr;
+};
+
+static char Hostname[1024];
+
+static std::string
+getPreBody(TSHttpTxn txn)
+{
+ std::stringstream output;
+ output << "{'xDebugProbeAt' : '" << Hostname << "'\n 'captured':[";
+ print_request_headers(txn, output);
+ output << "\n ]\n}";
+ output << MultipartBoundary;
+ return output.str();
+}
+
+static std::string
+getPostBody(TSHttpTxn txn)
+{
+ std::stringstream output;
+ output << MultipartBoundary;
+ output << "{'xDebugProbeAt' : '" << Hostname << "'\n 'captured':[";
+ print_response_headers(txn, output);
+ output << "\n ]\n}";
+ return output.str();
+}
+
+static void
+writePostBody(BodyBuilder *data)
+{
+ if (data->wrote_body && data->hdr_ready && !data->wrote_postbody.test_and_set()) {
+ TSDebug("xdebug_transform", "body_transform(): Writing postbody headers...");
+ std::string postbody = getPostBody(data->txn);
+ TSIOBufferWrite(data->output_buffer, postbody.data(), postbody.length());
+ data->nbytes += postbody.length();
+ TSVIONBytesSet(data->output_vio, data->nbytes);
+ TSVIOReenable(data->output_vio);
+ }
+}
+
+static int
+body_transform(TSCont contp, TSEvent event, void *edata)
+{
+ BodyBuilder *data = static_cast<BodyBuilder *>(TSContDataGet(contp));
+ if (!data) {
+ TSContDestroy(contp);
+ return TS_ERROR;
+ }
+ if (TSVConnClosedGet(contp)) {
+ // write connection destoried. cleanup.
+ delete data;
+ TSContDestroy(contp);
+ return 0;
+ }
+
+ TSVIO src_vio = TSVConnWriteVIOGet(contp);
+
+ switch (event) {
+ case TS_EVENT_ERROR: {
+ // Notify input vio of this error event
+ TSContCall(TSVIOContGet(src_vio), TS_EVENT_ERROR, src_vio);
+ return 0;
+ }
+ case TS_EVENT_VCONN_WRITE_COMPLETE: {
+ TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
+ return 0;
+ }
+ case TS_EVENT_VCONN_WRITE_READY:
+ TSDebug("xdebug_transform", "body_transform(): Event is TS_EVENT_VCONN_WRITE_READY");
+ // fallthru
+ default:
+ if (!data->output_buffer) {
+ data->output_buffer = TSIOBufferCreate();
+ data->output_reader = TSIOBufferReaderAlloc(data->output_buffer);
+ data->output_vio = TSVConnWrite(TSTransformOutputVConnGet(contp), contp, data->output_reader, INT64_MAX);
+ }
+
+ if (data->wrote_prebody == false) {
+ TSDebug("xdebug_transform", "body_transform(): Writing prebody headers...");
+ std::string prebody = getPreBody(data->txn);
+ TSIOBufferWrite(data->output_buffer, prebody.data(), prebody.length()); // write prebody
+ data->wrote_prebody = true;
+ data->nbytes += prebody.length();
+ }
+
+ TSIOBuffer src_buf = TSVIOBufferGet(src_vio);
+
+ if (!src_buf) {
+ // upstream continuation shuts down write operation.
+ data->wrote_body = true;
+ writePostBody(data);
+ return 0;
+ }
+
+ int64_t towrite = TSVIONTodoGet(src_vio);
+ TSDebug("xdebug_transform", "body_transform(): %li bytes of body is expected", towrite);
+ int64_t avail = TSIOBufferReaderAvail(TSVIOReaderGet(src_vio));
+ towrite = towrite > avail ? avail : towrite;
+ if (towrite > 0) {
+ TSIOBufferCopy(TSVIOBufferGet(data->output_vio), TSVIOReaderGet(src_vio), towrite, 0);
+ TSIOBufferReaderConsume(TSVIOReaderGet(src_vio), towrite);
+ TSVIONDoneSet(src_vio, TSVIONDoneGet(src_vio) + towrite);
+ TSDebug("xdebug_transform", "body_transform(): writing %li bytes of body", towrite);
+ }
+
+ if (TSVIONTodoGet(src_vio) > 0) {
+ TSVIOReenable(data->output_vio);
+ TSContCall(TSVIOContGet(src_vio), TS_EVENT_VCONN_WRITE_READY, src_vio);
+ } else {
+ // End of src vio
+ // Write post body content and update output VIO
+ data->wrote_body = true;
+ data->nbytes += TSVIONDoneGet(src_vio);
+ writePostBody(data);
+ TSContCall(TSVIOContGet(src_vio), TS_EVENT_VCONN_WRITE_COMPLETE, src_vio);
+ }
+ }
+ return 0;
+}