You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficserver.apache.org by jp...@apache.org on 2014/11/13 18:20:38 UTC
[2/3] trafficserver git commit: TS-3195: add crash logging
infrastructure and helper
TS-3195: add crash logging infrastructure and helper
Add the concept of a crash logging helper that is invoked to generate
a detailed crash dump when traffic_server receives a fatal signal.
Add traffic_server infrastructure to start the helper and invoke
it at crash time.
Add traffic_crashlog, a default crash log helper. Document the
whole thing.
Project: http://git-wip-us.apache.org/repos/asf/trafficserver/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafficserver/commit/fb8ba600
Tree: http://git-wip-us.apache.org/repos/asf/trafficserver/tree/fb8ba600
Diff: http://git-wip-us.apache.org/repos/asf/trafficserver/diff/fb8ba600
Branch: refs/heads/master
Commit: fb8ba600251478d13c316c4d520e90dd02f2f921
Parents: ecea941
Author: James Peach <jp...@apache.org>
Authored: Tue Nov 4 12:42:58 2014 -0800
Committer: James Peach <jp...@apache.org>
Committed: Thu Nov 13 09:18:37 2014 -0800
----------------------------------------------------------------------
cmd/Makefile.am | 1 +
cmd/traffic_crashlog/Makefile.am | 42 +++
cmd/traffic_crashlog/procinfo.cc | 321 +++++++++++++++++++
cmd/traffic_crashlog/traffic_crashlog.cc | 213 ++++++++++++
cmd/traffic_crashlog/traffic_crashlog.h | 58 ++++
configure.ac | 1 +
doc/manpages.py | 1 +
doc/reference/commands/index.en.rst | 1 +
doc/reference/commands/traffic_crashlog.en.rst | 94 ++++++
.../configuration/records.config.en.rst | 14 +
lib/ts/TextBuffer.cc | 21 +-
lib/ts/TextBuffer.h | 5 +
lib/ts/ink_config.h.in | 2 +
mgmt/RecordsConfig.cc | 8 +
proxy/Crash.cc | 155 +++++++++
proxy/Main.cc | 10 +-
proxy/Main.h | 3 +
proxy/Makefile.am | 1 +
18 files changed, 944 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/cmd/Makefile.am
----------------------------------------------------------------------
diff --git a/cmd/Makefile.am b/cmd/Makefile.am
index 80a13e2..5f02dd5 100644
--- a/cmd/Makefile.am
+++ b/cmd/Makefile.am
@@ -17,6 +17,7 @@
SUBDIRS = \
traffic_cop \
+ traffic_crashlog \
traffic_layout \
traffic_line \
traffic_manager \
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/cmd/traffic_crashlog/Makefile.am
----------------------------------------------------------------------
diff --git a/cmd/traffic_crashlog/Makefile.am b/cmd/traffic_crashlog/Makefile.am
new file mode 100644
index 0000000..a51f1cd
--- /dev/null
+++ b/cmd/traffic_crashlog/Makefile.am
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin_PROGRAMS = traffic_crashlog
+
+AM_CPPFLAGS = \
+ $(iocore_include_dirs) \
+ -I$(top_srcdir)/lib \
+ -I$(top_builddir)/lib \
+ -I$(top_srcdir)/lib/ts \
+ -I$(top_srcdir)/lib/records \
+ -I$(top_srcdir)/mgmt \
+ -I$(top_srcdir)/mgmt/utils \
+ -I$(top_srcdir)/mgmt/api/include
+
+traffic_crashlog_SOURCES = \
+ procinfo.cc \
+ traffic_crashlog.cc \
+ traffic_crashlog.h
+
+traffic_crashlog_LDADD = \
+ $(top_builddir)/lib/records/librecords_p.a \
+ $(top_builddir)/mgmt/libmgmt_p.la \
+ $(top_builddir)/iocore/eventsystem/libinkevent.a \
+ $(top_builddir)/proxy/shared/libUglyLogStubs.a \
+ $(top_builddir)/mgmt/api/libtsmgmt.la \
+ $(top_builddir)/lib/ts/libtsutil.la \
+ @LIBTCL@ @HWLOC_LIBS@
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/cmd/traffic_crashlog/procinfo.cc
----------------------------------------------------------------------
diff --git a/cmd/traffic_crashlog/procinfo.cc b/cmd/traffic_crashlog/procinfo.cc
new file mode 100644
index 0000000..979c86e
--- /dev/null
+++ b/cmd/traffic_crashlog/procinfo.cc
@@ -0,0 +1,321 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "traffic_crashlog.h"
+#include <sys/utsname.h>
+
+// ucontext.h is deprecated on Darwin, and we really only need it on Linux, so only
+// include it if we are planning to use it.
+#if defined(__linux__)
+#include <ucontext.h>
+#endif
+
+static int
+procfd_open(pid_t pid, const char * fname)
+{
+ char path[128];
+ snprintf(path, sizeof(path), "/proc/%ld/%s", (long)pid, fname);
+ return open(path, O_RDONLY);
+}
+
+static char *
+procfd_readlink(pid_t pid, const char * fname)
+{
+ char path[128];
+ ssize_t nbytes;
+ ats_scoped_str resolved((char *)ats_malloc(MAXPATHLEN + 1));
+
+ snprintf(path, sizeof(path), "/proc/%ld/%s", (long)pid, fname);
+ nbytes = readlink(path, resolved, MAXPATHLEN);
+ if (nbytes == -1) {
+ Note("readlink failed with %s", strerror(errno));
+ return NULL;
+ }
+
+ resolved[nbytes] = '\0';
+ return resolved.release();
+}
+
+bool
+crashlog_write_regions(FILE * fp, const crashlog_target& target)
+{
+ ats_scoped_fd fd;
+ textBuffer text(0);
+
+ fd = procfd_open(target.pid, "maps");
+ if (fd != -1) {
+ text.slurp(fd);
+ text.chomp();
+ fprintf(fp, "Memory Regions:\n%.*s\n", (int)text.spaceUsed(), text.bufPtr());
+ }
+
+ return !text.empty();
+}
+
+bool
+crashlog_write_uname(FILE * fp, const crashlog_target&)
+{
+ struct utsname uts;
+
+ if (uname(&uts) == 0) {
+ fprintf(fp, LABELFMT "%s %s %s %s\n", "System Version:", uts.sysname, uts.machine, uts.version, uts.release);
+ } else {
+ fprintf(fp, LABELFMT "%s\n", "System Version:", "unknown");
+ }
+
+ return true;
+}
+
+bool
+crashlog_write_exename(FILE * fp, const crashlog_target& target)
+{
+ ats_scoped_str str;
+
+ str = procfd_readlink(target.pid, "exe");
+ if (str) {
+ fprintf(fp, LABELFMT "%s\n", "File:", (const char *)str);
+ return true;
+ }
+
+ return false;
+}
+
+bool
+crashlog_write_procname(FILE * fp, const crashlog_target& target)
+{
+ ats_scoped_fd fd;
+ ats_scoped_str str;
+ textBuffer text(0);
+
+ fd = procfd_open(target.pid, "comm");
+ if (fd != -1) {
+ text.slurp(fd);
+ text.chomp();
+ fprintf(fp, LABELFMT "%s [%ld]\n", "Process:", text.bufPtr(), (long)target.pid);
+ } else {
+ fprintf(fp, LABELFMT "%ld\n", "Process:", (long)target.pid);
+ }
+
+ return true;
+}
+
+bool
+crashlog_write_datime(FILE * fp, const crashlog_target& target)
+{
+ char buf[128];
+
+ strftime(buf, sizeof(buf), "%a, %d %b %Y %T %z", &target.timestamp);
+ fprintf(fp, LABELFMT "%s\n", "Date:", buf);
+ return true;
+}
+
+bool
+crashlog_write_procstatus(FILE * fp, const crashlog_target& target)
+{
+ ats_scoped_fd fd;
+ textBuffer text(0);
+
+ fd = procfd_open(target.pid, "status");
+ if (fd != -1) {
+ text.slurp(fd);
+ text.chomp();
+
+ fprintf(fp, "Process Status:\n%s\n", text.bufPtr());
+ }
+
+ return !text.empty();
+}
+
+bool
+crashlog_write_backtrace(FILE * fp, const crashlog_target&)
+{
+ TSString trace = NULL;
+ TSMgmtError mgmterr;
+
+ // NOTE: sometimes we can't get a backtrace because the ptrace attach will fail with
+ // EPERM. I've seen this happen when a debugger is attached, which makes sense, but it
+ // can also happen without a debugger. Possibly in that case, there is a race with the
+ // kernel locking the process information?
+
+ if ((mgmterr = TSProxyBacktraceGet(0, &trace)) != TS_ERR_OKAY) {
+ char * msg = TSGetErrorMessage(mgmterr);
+ fprintf(fp, "Unable to retrieve backtrace: %s\n", msg);
+ TSfree(msg);
+ return false;
+ }
+
+ fprintf(fp, "%s", trace);
+ TSfree(trace);
+ return true;
+}
+
+bool
+crashlog_write_records(FILE * fp, const crashlog_target&)
+{
+ TSMgmtError mgmterr;
+ TSList list = TSListCreate();
+ bool success = false;
+
+ if ((mgmterr = TSRecordGetMatchMlt(".", list)) != TS_ERR_OKAY) {
+ char * msg = TSGetErrorMessage(mgmterr);
+ fprintf(fp, "Unable to retrieve Traffic Server records: %s\n", msg);
+ TSfree(msg);
+ goto done;
+ }
+
+ // If the RPC call failed, the list will be empty, so we won't print anything. Otherwise,
+ // print all the results, freeing them as we go.
+ for (TSRecordEle * rec_ele = (TSRecordEle *) TSListDequeue(list); rec_ele;
+ rec_ele = (TSRecordEle *) TSListDequeue(list)) {
+
+ if (!success) {
+ success = true;
+ fprintf(fp, "Traffic Server Configuration Records:\n");
+ }
+
+ switch (rec_ele->rec_type) {
+ case TS_REC_INT:
+ fprintf(fp, "%s %" PRId64 "\n", rec_ele->rec_name, rec_ele->valueT.int_val);
+ break;
+ case TS_REC_COUNTER:
+ fprintf(fp, "%s %" PRId64 "\n", rec_ele->rec_name, rec_ele->valueT.counter_val);
+ break;
+ case TS_REC_FLOAT:
+ fprintf(fp, "%s %f\n", rec_ele->rec_name, rec_ele->valueT.float_val);
+ break;
+ case TS_REC_STRING:
+ fprintf(fp, "%s %s\n", rec_ele->rec_name, rec_ele->valueT.string_val);
+ break;
+ default:
+ // just skip it ...
+ break;
+ }
+
+ TSRecordEleDestroy(rec_ele);
+ }
+
+done:
+ TSListDestroy(list);
+ return success;
+}
+
+bool
+crashlog_write_siginfo(FILE * fp, const crashlog_target& target)
+{
+ char tmp[32];
+
+ if (!(CRASHLOG_HAVE_THREADINFO & target.flags)) {
+ fprintf(fp, "No target signal information\n");
+ return false;
+ }
+
+ fprintf(fp, "Signal Status:\n");
+ fprintf(fp, LABELFMT "%d (%s)\n", "siginfo.si_signo:",
+ target.siginfo.si_signo, strsignal(target.siginfo.si_signo));
+
+ snprintf(tmp, sizeof(tmp), "%ld", (long)target.siginfo.si_pid);
+ fprintf(fp, LABELFMT LABELFMT, "siginfo.si_pid:", tmp);
+ fprintf(fp, LABELFMT "%ld", "siginfo.si_uid:", (long)target.siginfo.si_uid);
+ fprintf(fp, "\n");
+
+ snprintf(tmp, sizeof(tmp), "0x%x (%d)", target.siginfo.si_code, target.siginfo.si_code);
+ fprintf(fp, LABELFMT LABELFMT, "siginfo.si_code:", tmp);
+ fprintf(fp, LABELFMT ADDRFMT, "siginfo.si_addr:", (long long)target.siginfo.si_addr);
+ fprintf(fp, "\n");
+
+ if (target.siginfo.si_code == SI_USER) {
+ fprintf(fp, "Signal delivered by user %ld from process %ld\n",
+ (long)target.siginfo.si_uid, (long)target.siginfo.si_pid);
+ return true;
+ }
+
+ if (target.siginfo.si_signo == SIGSEGV) {
+ const char * msg = "Unknown error";
+
+ switch (target.siginfo.si_code) {
+ case SEGV_MAPERR: msg = "No object mapped"; break;
+ case SEGV_ACCERR: msg = "Invalid permissions for mapped object"; break;
+ }
+
+ fprintf(fp, "%s at address " ADDRFMT "\n", msg, (long long)target.siginfo.si_addr);
+ return true;
+ }
+
+ if (target.siginfo.si_signo == SIGSEGV) {
+ const char * msg = "Unknown error";
+
+ switch (target.siginfo.si_code) {
+ case BUS_ADRALN: msg = "Invalid address alignment"; break;
+ case BUS_ADRERR: msg = "Nonexistent physical address"; break;
+ case BUS_OBJERR: msg = "Object-specific hardware error"; break;
+ }
+
+ fprintf(fp, "%s at address " ADDRFMT "\n", msg, (long long)target.siginfo.si_addr);
+ return true;
+ }
+
+ return true;
+}
+
+bool
+crashlog_write_registers(FILE * fp, const crashlog_target& target)
+{
+ if (!(CRASHLOG_HAVE_THREADINFO & target.flags)) {
+ fprintf(fp, "No target CPU registers\n");
+ return false;
+ }
+
+#if defined(__linux__) && (defined(__x86_64__) || defined(__i386__))
+
+ // x86 register names as per ucontext.h.
+#if defined(__i386__)
+#define REGFMT "0x%08x"
+ static const char * names[NGREG] = {
+ "GS", "FS", "ES", "DS", "EDI", "ESI", "EBP", "ESP",
+ "EBX", "EDX", "ECX", "EAX", "TRAPNO", "ERR", "EIP", "CS",
+ "EFL", "UESP", "SS"
+ };
+#endif
+
+#if defined(__x86_64__)
+#define REGFMT "0x%016llx"
+ static const char * names[NGREG] = {
+ "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15",
+ "RDI", "RSI", "RBP", "RBX", "RDX", "RAX", "RCX", "RSP",
+ "RIP", "EFL", "CSGSFS", "ERR", "TRAPNO", "OLDMASK", "CR2"
+ };
+#endif
+
+ fprintf(fp, "CPU Registers:\n");
+ for (unsigned i = 0; i < countof(names); ++i) {
+ const char * trailer = ((i % 4) == 3) ? "\n" : " ";
+ fprintf(fp, "%-3s:" REGFMT "%s", names[i], target.ucontext.uc_mcontext.gregs[i], trailer);
+ }
+
+ fprintf(fp, "\n");
+ return true;
+#else
+ fprintf(fp, "No target CPU register support on this architecture\n");
+ return false;
+#endif
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/cmd/traffic_crashlog/traffic_crashlog.cc
----------------------------------------------------------------------
diff --git a/cmd/traffic_crashlog/traffic_crashlog.cc b/cmd/traffic_crashlog/traffic_crashlog.cc
new file mode 100644
index 0000000..0005c81
--- /dev/null
+++ b/cmd/traffic_crashlog/traffic_crashlog.cc
@@ -0,0 +1,213 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "traffic_crashlog.h"
+#include "ink_args.h"
+#include "ink_cap.h"
+#include "I_Version.h"
+#include "I_Layout.h"
+#include "I_RecProcess.h"
+#include "RecordsConfig.h"
+
+static int syslog_mode = false;
+static int debug_mode = false;
+static int wait_mode = false;
+static char * host_triplet = NULL;
+static int target_pid = getppid();
+
+// If pid_t is not sizeof(int), we will have to jiggle argument parsing.
+extern char __pid_size_static_assert[sizeof(pid_t) == sizeof(int) ? 0 : -1];
+
+static AppVersionInfo appVersionInfo;
+static const ArgumentDescription argument_descriptions[] = {
+ { "target", '-', "Target process ID", "I", &target_pid, NULL, NULL },
+ { "host", '-', "Host triplet for the process being logged", "S*", &host_triplet, NULL, NULL },
+ { "wait", '-', "Stop until signalled at startup", "F", &wait_mode, NULL, NULL },
+ { "syslog", '-', "Syslog after writing a crash log", "F", &syslog_mode, NULL, NULL },
+ { "debug", '-', "Enable debugging mode", "F", &debug_mode, NULL, NULL },
+ HELP_ARGUMENT_DESCRIPTION(),
+ VERSION_ARGUMENT_DESCRIPTION()
+};
+
+static struct tm
+timestamp()
+{
+ time_t now = time(NULL);
+ struct tm tm;
+
+ localtime_r(&now, &tm);
+ return tm;
+}
+
+static char *
+crashlog_name()
+{
+ char filename[64];
+ struct tm now = timestamp();
+ ats_scoped_str logdir(RecConfigReadLogDir());
+ ats_scoped_str pathname;
+
+ strftime(filename, sizeof(filename), "crash-%Y-%m-%d-%H%M%S.log", &now);
+ pathname = Layout::relative_to(logdir, filename);
+
+ return pathname.release();
+}
+
+static FILE *
+crashlog_open(const char * path)
+{
+ int fd;
+
+ fd = open(path, O_WRONLY | O_TRUNC, 0400);
+ return (fd == -1) ? NULL : fdopen(fd, "w");
+}
+
+int
+main(int /* argc ATS_UNUSED */, char **argv)
+{
+ FILE * fp;
+ char * logname;
+ TSMgmtError mgmterr;
+ crashlog_target target;
+
+ diags = new Diags("" /* tags */, "" /* actions */, stderr);
+
+ appVersionInfo.setup(PACKAGE_NAME, "traffic_crashlog", PACKAGE_VERSION,
+ __DATE__, __TIME__, BUILD_MACHINE, BUILD_PERSON, "");
+
+ // Process command line arguments and dump into variables
+ process_args(&appVersionInfo, argument_descriptions, countof(argument_descriptions), argv);
+
+ if (wait_mode) {
+ EnableDeathSignal(SIGKILL);
+ kill(getpid(), SIGSTOP);
+ }
+
+ // XXX This is a hack. traffic_manager starts traffic_server with the euid of the admin user. We are still
+ // privileged, but won't be able to open files in /proc or ptrace the target. This really should be fixed
+ // in traffic_manager.
+ if (getuid() == 0) {
+ seteuid(0);
+ }
+
+ Layout::create();
+ RecProcessInit(RECM_STAND_ALONE, NULL /* diags */);
+ LibRecordsConfigInit();
+ RecordsConfigOverrideFromEnvironment();
+
+ if (syslog_mode) {
+ RecString name;
+ int facility = -1;
+
+ if (REC_ReadConfigStringAlloc(name, "proxy.config.syslog_facility") == REC_ERR_OKAY) {
+ facility = facility_string_to_int(name);
+ ats_free(name);
+ }
+
+ if (facility < 0) {
+ facility = LOG_DAEMON;
+ }
+
+ openlog(appVersionInfo.AppStr, LOG_PID | LOG_NDELAY | LOG_NOWAIT, facility);
+ diags->config.outputs[DL_Debug].to_syslog = true;
+ diags->config.outputs[DL_Status].to_syslog = true;
+ diags->config.outputs[DL_Note].to_syslog = true;
+ diags->config.outputs[DL_Warning].to_syslog = true;
+ diags->config.outputs[DL_Error].to_syslog = true;
+ diags->config.outputs[DL_Fatal].to_syslog = true;
+ diags->config.outputs[DL_Alert].to_syslog = true;
+ diags->config.outputs[DL_Emergency].to_syslog = true;
+ }
+
+ Note("crashlog started, target=%ld, debug=%s syslog=%s, uid=%ld euid=%ld",
+ (long)target_pid, debug_mode ? "true" : "false", syslog_mode ? "true" : "false",
+ (long)getuid(), (long)geteuid());
+
+ mgmterr = TSInit(NULL, (TSInitOptionT)(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS));
+ if (mgmterr != TS_ERR_OKAY) {
+ char * msg = TSGetErrorMessage(mgmterr);
+ Warning("failed to intialize management API: %s", msg);
+ TSfree(msg);
+ }
+
+ ink_zero(target);
+ target.pid = (pid_t)target_pid;
+ target.timestamp = timestamp();
+
+ if (host_triplet && strncmp(host_triplet, "x86_64-unknown-linux", sizeof("x86_64-unknown-linux") - 1) == 0) {
+ ssize_t nbytes;
+ target.flags |= CRASHLOG_HAVE_THREADINFO;
+
+ nbytes = read(STDIN_FILENO, &target.siginfo, sizeof(target.siginfo));
+ if (nbytes < (ssize_t)sizeof(target.siginfo)) {
+ Warning("received %zd of %zu expected signal info bytes", nbytes, sizeof(target.siginfo));
+ target.flags &= ~CRASHLOG_HAVE_THREADINFO;
+ }
+
+ nbytes = read(STDIN_FILENO, &target.ucontext, sizeof(target.ucontext));
+ if (nbytes < (ssize_t)sizeof(target.ucontext)) {
+ Warning("received %zd of %zu expected thread context bytes", nbytes, sizeof(target.ucontext));
+ target.flags &= ~CRASHLOG_HAVE_THREADINFO;
+ }
+ }
+
+ logname = crashlog_name();
+
+ fp = debug_mode ? stdout : crashlog_open(logname);
+ if (fp == NULL) {
+ Error("failed to create '%s': %s", logname, strerror(errno));
+ return 1;
+ }
+
+ Note("logging to %p", fp);
+
+ crashlog_write_procname(fp, target);
+ crashlog_write_exename(fp, target);
+ fprintf(fp, LABELFMT "Traffic Server %s\n", "Version:", PACKAGE_VERSION);
+ crashlog_write_uname(fp, target);
+ crashlog_write_datime(fp, target);
+
+ fprintf(fp, "\n");
+ crashlog_write_siginfo(fp, target);
+
+ fprintf(fp, "\n");
+ crashlog_write_registers(fp, target);
+
+ fprintf(fp, "\n");
+ crashlog_write_backtrace(fp, target);
+
+ fprintf(fp, "\n");
+ crashlog_write_procstatus(fp, target);
+
+ fprintf(fp, "\n");
+ crashlog_write_regions(fp, target);
+
+ fprintf(fp, "\n");
+ crashlog_write_records(fp, target);
+
+ Error("wrote crash log to %s", logname);
+
+ fflush(fp);
+ fclose(fp);
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/cmd/traffic_crashlog/traffic_crashlog.h
----------------------------------------------------------------------
diff --git a/cmd/traffic_crashlog/traffic_crashlog.h b/cmd/traffic_crashlog/traffic_crashlog.h
new file mode 100644
index 0000000..9b57ee3
--- /dev/null
+++ b/cmd/traffic_crashlog/traffic_crashlog.h
@@ -0,0 +1,58 @@
+/** @file
+
+ A brief file description
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#ifndef __TRAFFIC_CRASHLOG_H__
+#define __TRAFFIC_CRASHLOG_H__
+
+#include "libts.h"
+#include "mgmtapi.h"
+
+// Printf format for crash log field labels.
+#define LABELFMT "%-20s"
+
+// Printf format for memory addresses.
+#define ADDRFMT "0x%016llx"
+
+#define CRASHLOG_HAVE_THREADINFO 0x1u
+
+struct crashlog_target
+{
+ pid_t pid;
+ siginfo_t siginfo;
+ ucontext_t ucontext;
+ struct tm timestamp;
+ unsigned flags;
+};
+
+bool crashlog_write_backtrace(FILE *, const crashlog_target&);
+bool crashlog_write_regions(FILE * , const crashlog_target&);
+bool crashlog_write_exename(FILE *, const crashlog_target&);
+bool crashlog_write_uname(FILE *, const crashlog_target&);
+bool crashlog_write_datime(FILE *, const crashlog_target&);
+bool crashlog_write_procname(FILE *, const crashlog_target&);
+bool crashlog_write_procstatus(FILE *, const crashlog_target&);
+bool crashlog_write_records(FILE *, const crashlog_target&);
+bool crashlog_write_siginfo(FILE *, const crashlog_target&);
+bool crashlog_write_registers(FILE *, const crashlog_target&);
+
+#endif /* __TRAFFIC_CRASHLOG_H__ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 6e8cf4f..3e4465b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1853,6 +1853,7 @@ AC_CONFIG_FILES([
Makefile
cmd/Makefile
cmd/traffic_cop/Makefile
+ cmd/traffic_crashlog/Makefile
cmd/traffic_layout/Makefile
cmd/traffic_line/Makefile
cmd/traffic_manager/Makefile
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/doc/manpages.py
----------------------------------------------------------------------
diff --git a/doc/manpages.py b/doc/manpages.py
index ca6b6e8..0361671 100644
--- a/doc/manpages.py
+++ b/doc/manpages.py
@@ -22,6 +22,7 @@ man_pages = [
('reference/api/' + filename[:-4], filename.split('.', 1)[0], '', None, '3ts') for filename in os.listdir('reference/api') if filename != 'index.en.rst' and filename.endswith('.rst')] + [
('reference/commands/traffic_cop.en', 'traffic_cop', u'Traffic Server watchdog', None, '8'),
+ ('reference/commands/traffic_crashlog.en', 'traffic_crashlog', u'Traffic Server crash log helper', None, '8'),
('reference/commands/traffic_line.en', 'traffic_line', u'Traffic Server command line', None, '8'),
('reference/commands/traffic_logcat.en', 'traffic_logcat', u'Traffic Server log spooler', None, '8'),
('reference/commands/traffic_logstats.en', 'traffic_logstats', u'Traffic Server analyzer', None, '8'),
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/doc/reference/commands/index.en.rst
----------------------------------------------------------------------
diff --git a/doc/reference/commands/index.en.rst b/doc/reference/commands/index.en.rst
index 6f42d65..a6b615b 100644
--- a/doc/reference/commands/index.en.rst
+++ b/doc/reference/commands/index.en.rst
@@ -22,6 +22,7 @@ Command Reference
:maxdepth: 1
traffic_cop.en
+ traffic_crashlog.en
traffic_line.en
traffic_logcat.en
traffic_logstats.en
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/doc/reference/commands/traffic_crashlog.en.rst
----------------------------------------------------------------------
diff --git a/doc/reference/commands/traffic_crashlog.en.rst b/doc/reference/commands/traffic_crashlog.en.rst
new file mode 100644
index 0000000..49ba9b3
--- /dev/null
+++ b/doc/reference/commands/traffic_crashlog.en.rst
@@ -0,0 +1,94 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+================
+traffic_crashlog
+================
+
+Synopsis
+========
+
+
+:program:`traffic_crashlog` [options]
+
+.. _traffic-crashlog-commands:
+
+Description
+===========
+
+:program:`traffic_crashlog` is a helper process that catches Traffic Server
+crashes and writes a crash report log to the logging directory. Other than for
+debugging or development purposes, :program:`traffic_crashlog` is not intended
+for users to run directly.
+
+When :program:`traffic_server` starts, it will launch a
+:program:`traffic_crashlog` process and keep it stopped, activating
+it only if a crash occurs.
+
+Options
+=======
+
+.. program:: traffic_crashlog
+
+.. option:: --host TRIPLE
+
+ This option specifies the host triple for the process that
+ :program:`traffic_crashlog` should examine. If a supported host
+ triple is specified, :program:`traffic_crashlog` expects to
+ receive a ``siginfo_t`` structure on it's standard input,
+ followed by a ``ucontext_t``.
+
+.. option:: --target PID
+
+ Specifies the process ID of the crashing :program:`traffic_server`
+ process. If this option is not specified, :program:`traffic_crashlog`
+ assumes it should examine it's parent process.
+
+.. option:: --syslog
+
+ This option causes :program:`traffic_crashlog` to log the name
+ of the crash log it creates to the system log.
+
+.. option:: --debug
+
+ This option enables debugging mode. In this mode,
+ :program:`traffic_crashlog` emits the log to it's standard
+ output.
+
+.. option:: --wait
+
+ This option causes :program:`traffic_crashlog` to stop itself
+ immediately after it is launched. :program:`traffic_server`
+ will allow it to continue only when a crash occurs.
+
+Caveats
+=======
+
+:program:`traffic_crashlog` makes use of various Traffic Server management
+APIs. If :program:`traffic_manager` is not available, the crash log will be
+incomplete.
+
+:program:`traffic_crashlog` may generate a crash log containing information you
+would rather not share outside your organization. Please examine the crash log
+carefully before posting it in a publc forum.
+
+See also
+========
+
+:manpage:`records.config(5)`,
+:manpage:`traffic_manager(8)`,
+:manpage:`traffic_server(8)`
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/doc/reference/configuration/records.config.en.rst
----------------------------------------------------------------------
diff --git a/doc/reference/configuration/records.config.en.rst b/doc/reference/configuration/records.config.en.rst
index a69b7dd..ded0832 100644
--- a/doc/reference/configuration/records.config.en.rst
+++ b/doc/reference/configuration/records.config.en.rst
@@ -252,6 +252,20 @@ Value Effect
Set the maximum number of file handles for the traffic_server process as a percentage of the the fs.file-max proc value in Linux. The default is 90%.
+.. ts:cv:: CONFIG proxy.config.crash_log_helper STRING traffic_crashlog
+
+ This option directs :program:`traffic_server` to spawn a crash
+ log helper at startup. The value should be the path to an
+ executable program. If the path is not absolute, it is located
+ relative to configured ``bin`` directory. Any user-provided
+ program specified here must behave in a fashion compatible with
+ :program:`traffic_crashlog`. Specifically, it must implement
+ the :option:`traffic_crashlog --wait` behavior.
+
+ This setting not reloadable because the helper must be spawned
+ before :program:`traffic_server` drops privilege. If this variable
+ is set to ``NULL``, no helper will be spawned.
+
Network
=======
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/lib/ts/TextBuffer.cc
----------------------------------------------------------------------
diff --git a/lib/ts/TextBuffer.cc b/lib/ts/TextBuffer.cc
index 8c7c8a3..374892a 100644
--- a/lib/ts/TextBuffer.cc
+++ b/lib/ts/TextBuffer.cc
@@ -185,6 +185,17 @@ textBuffer::rawReadFromFile(int fd)
}
}
+// Read the entire contents of the given file descriptor.
+void
+textBuffer::slurp(int fd)
+{
+ int nbytes;
+
+ do {
+ nbytes = readFromFD(fd);
+ } while (nbytes > 0);
+}
+
// int textBuffer::readFromFD(int fd)
//
// Issues a single read command on the file
@@ -257,11 +268,9 @@ textBuffer::format(const char * fmt, ...)
void
textBuffer::chomp()
{
- if (nextAdd > bufferStart) {
- if (nextAdd[-1] == '\n') {
- --nextAdd;
- ++spaceLeft;
- *nextAdd = '\0';
- }
+ while ((nextAdd > bufferStart) && (nextAdd[-1] == '\n')) {
+ --nextAdd;
+ ++spaceLeft;
+ *nextAdd = '\0';
}
}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/lib/ts/TextBuffer.h
----------------------------------------------------------------------
diff --git a/lib/ts/TextBuffer.h b/lib/ts/TextBuffer.h
index ab2fa40..2dae20b 100644
--- a/lib/ts/TextBuffer.h
+++ b/lib/ts/TextBuffer.h
@@ -46,11 +46,16 @@ public:
void reUse();
inkcoreapi char *bufPtr();
+ void clear() { this->reUse(); }
+ void resize(unsigned nbytes) { this->enlargeBuffer(nbytes); }
+
size_t spaceUsed() const {
return (size_t) (nextAdd - bufferStart);
};
void chomp();
+ void slurp(int);
+ bool empty() const { return this->spaceUsed() == 0; }
void format(const char * fmt, ...) TS_PRINTFLIKE(2, 3);
char * release();
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/lib/ts/ink_config.h.in
----------------------------------------------------------------------
diff --git a/lib/ts/ink_config.h.in b/lib/ts/ink_config.h.in
index c9ca78f..9f07825 100644
--- a/lib/ts/ink_config.h.in
+++ b/lib/ts/ink_config.h.in
@@ -118,6 +118,8 @@
#define TS_BUILD_CACHEDIR "@rel_cachedir@"
#define TS_BUILD_INFODIR "@rel_infodir@"
+#define TS_BUILD_CANONICAL_HOST "@host@"
+
#define TS_BUILD_DEFAULT_LOOPBACK_IFACE "@default_loopback_iface@"
#endif /* _ink_config_h */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/mgmt/RecordsConfig.cc
----------------------------------------------------------------------
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index e6a0ad8..ad6e5cb 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -26,6 +26,12 @@
#include "RecordsConfig.h"
#include "ParseRules.h"
+#if TS_USE_REMOTE_UNWINDING
+#define MGMT_CRASHLOG_HELPER "traffic_crashlog"
+#else
+#define MGMT_CRASHLOG_HELPER NULL
+#endif
+
//-------------------------------------------------------------------------
// RecordsConfig
//-------------------------------------------------------------------------
@@ -66,6 +72,8 @@ RecordElement RecordsConfig[] = {
//# Negative core limit means max out limit
{RECT_CONFIG, "proxy.config.core_limit", RECD_INT, "-1", RECU_NULL, RR_NULL, RECC_NULL, NULL, RECA_NULL}
,
+ {RECT_CONFIG, "proxy.config.crash_log_helper", RECD_STRING, MGMT_CRASHLOG_HELPER, RECU_RESTART_TS, RR_NULL, RECC_NULL, NULL, RECA_NULL}
+ ,
// 0 - Disabled, 1 - enabled for important pages (e.g. cache directory), 2 - enabled for all pages
{RECT_CONFIG, "proxy.config.mlock_enabled", RECD_INT, "0", RECU_NULL, RR_NULL, RECC_NULL, NULL, RECA_NULL}
,
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/proxy/Crash.cc
----------------------------------------------------------------------
diff --git a/proxy/Crash.cc b/proxy/Crash.cc
new file mode 100644
index 0000000..83f917c
--- /dev/null
+++ b/proxy/Crash.cc
@@ -0,0 +1,155 @@
+/** @file
+
+ Crash logging helper support
+
+ @section license License
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+#include "Main.h"
+#include "I_Layout.h"
+#include "I_Net.h"
+#include "signals.h"
+#include "ink_cap.h"
+
+static pid_t crash_logger_pid = -1;
+static int crash_logger_fd = NO_FD;
+
+static char *
+create_logger_path()
+{
+ RecString name;
+ ats_scoped_str bindir;
+ ats_scoped_str fullpath;
+
+ if (RecGetRecordString_Xmalloc("proxy.config.crash_log_helper", &name) != REC_ERR_OKAY) {
+ return NULL;
+ }
+
+ // Take an absolute path as it is ...
+ if (name == NULL || *name == '/') {
+ return name;
+ }
+
+ // Otherwise locate it relative to $BINDIR.
+ bindir = RecConfigReadBinDir();
+ fullpath = Layout::relative_to(bindir, name);
+
+ ats_free(name);
+ return fullpath.release();
+}
+
+static bool
+check_logger_path(const char * path)
+{
+ struct stat sbuf;
+
+ if (stat(path, &sbuf) != 0) {
+ Error("failed to access crash log helper '%s': %s", path, strerror(errno));
+ return false;
+ }
+
+ if (!S_ISREG(sbuf.st_mode) || access(path, X_OK) != 0) {
+ Error("crash log helper '%s' is not executable", path);
+ return false;
+ }
+
+ return true;
+}
+
+void
+crash_logger_init()
+{
+ ats_scoped_str logger(create_logger_path());
+ const char * basename;
+
+ pid_t child;
+ int status;
+ int pipe[2];
+
+ // Do nothing the log helper was set to NULL, or we can't find it.
+ if (!logger || !check_logger_path(logger)) {
+ return;
+ }
+
+ // By this point, we have an absolute path, so we'd better be able to find the basename.
+ basename = strrchr(logger, '/') + 1;
+
+ socketpair(AF_UNIX, SOCK_STREAM, 0, pipe);
+
+ child = fork();
+ switch (child) {
+ case -1:
+ Error("failed to fork crash log helper: %s", strerror(errno));
+ crash_logger_pid = -1;
+ crash_logger_fd = NO_FD;
+ return;
+
+ case 0:
+ // Dupe the child socket to stdin.
+ dup2(pipe[1], STDIN_FILENO);
+ close(pipe[0]);
+ close(pipe[1]);
+ ink_release_assert(execl(logger, basename,
+ "--syslog", "--wait", "--host", TS_BUILD_CANONICAL_HOST, NULL) != -1);
+ return; // not reached.
+ }
+
+ close(pipe[1]);
+ crash_logger_pid = child;
+ crash_logger_fd = pipe[0];
+
+ // Wait for the helper to stop
+ if (waitpid(crash_logger_pid, &status, WUNTRACED) > 0) {
+ Debug("server", "waited on PID %ld, %s", (long)crash_logger_pid, WIFSTOPPED(status) ? "STOPPED" : "???");
+
+ if (WIFEXITED(status)) {
+ Warning("crash logger '%s' unexpectedly exited with status %d", (const char *)logger, WEXITSTATUS(status));
+ close(crash_logger_pid);
+ crash_logger_pid = -1;
+ crash_logger_fd = NO_FD;
+ }
+ }
+}
+
+void
+crash_logger_invoke(int signo, siginfo_t * info, void * ctx)
+{
+ int status;
+ ucontext_t * uctx = (ucontext_t *)ctx;
+
+ if (crash_logger_pid != -1) {
+ // Let the crash logger free ...
+ kill(crash_logger_pid, SIGCONT);
+
+ // Write the crashing thread information to the crash logger. While the siginfo_t is blesses by POSIX, the
+ // ucontext_t can contain pointers, so it's highly platform dependent. On Linux with glibc, however, it is
+ // a single memory block that we can just puke out.
+ write(crash_logger_fd, info, sizeof(siginfo_t));
+ write(crash_logger_fd, uctx, sizeof(ucontext_t));
+
+ close(crash_logger_fd);
+
+ // Wait for the logger to finish ...
+ waitpid(crash_logger_pid, &status, 0);
+ }
+
+ // Log the signal, dump a stack trace and core.
+ signal_format_siginfo(signo, info, appVersionInfo.AppStr); // XXX Add timestamp ...
+ signal_crash_handler(signo, info, ctx);
+}
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/proxy/Main.cc
----------------------------------------------------------------------
diff --git a/proxy/Main.cc b/proxy/Main.cc
index beb82ee..650ab0d 100644
--- a/proxy/Main.cc
+++ b/proxy/Main.cc
@@ -1367,6 +1367,7 @@ main(int /* argc ATS_UNUSED */, char **argv)
chdir_root(); // change directory to the install root of traffic server.
process_args(&appVersionInfo, argument_descriptions, countof(argument_descriptions), argv);
+ command_flag = command_flag || *command_string;
// Set stdout/stdin to be unbuffered
setbuf(stdout, NULL);
@@ -1417,6 +1418,14 @@ main(int /* argc ATS_UNUSED */, char **argv)
if (!num_task_threads)
REC_ReadConfigInteger(num_task_threads, "proxy.config.task_threads");
+ // Set up crash logging. We need to do this while we are still privileged so that the crash
+ // logging helper runs as root. Don't bother setting up a crash logger if we are going into
+ // command mode since that's not going to daemonize or run for a long time unattended.
+ if (!command_flag) {
+ crash_logger_init();
+ signal_register_crash_handler(crash_logger_invoke);
+ }
+
ats_scoped_str user(MAX_LOGIN + 1);
*user = '\0';
@@ -1509,7 +1518,6 @@ main(int /* argc ATS_UNUSED */, char **argv)
// Sanity checks
check_fd_limit();
- command_flag = command_flag || *command_string;
// Alter the frequecies at which the update threads will trigger
#define SET_INTERVAL(scope, name, var) do { \
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/proxy/Main.h
----------------------------------------------------------------------
diff --git a/proxy/Main.h b/proxy/Main.h
index 1a9e27b..3de3550 100644
--- a/proxy/Main.h
+++ b/proxy/Main.h
@@ -78,4 +78,7 @@ maintainance_mode()
extern AppVersionInfo appVersionInfo;
+void crash_logger_init();
+void crash_logger_invoke(int signo, siginfo_t * info, void * ctx);
+
#endif /* _Main_h_ */
http://git-wip-us.apache.org/repos/asf/trafficserver/blob/fb8ba600/proxy/Makefile.am
----------------------------------------------------------------------
diff --git a/proxy/Makefile.am b/proxy/Makefile.am
index 34ae17b..a1a18a6 100644
--- a/proxy/Makefile.am
+++ b/proxy/Makefile.am
@@ -135,6 +135,7 @@ traffic_server_SOURCES = \
ControlMatcher.h \
CoreUtils.cc \
CoreUtils.h \
+ Crash.cc \
DynamicStats.h \
EventName.cc \
FetchSM.cc \