You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by oc...@apache.org on 2020/04/15 17:05:18 UTC
[trafficcontrol] branch master updated: Debug TM, TO, TO Perl,
or TR CDN-in-a-Box containers (#4375)
This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new 609c3b8 Debug TM, TO, TO Perl, or TR CDN-in-a-Box containers (#4375)
609c3b8 is described below
commit 609c3b855a110767e245f354c456720600d67217
Author: Zach Hoffman <za...@zrhoffman.net>
AuthorDate: Wed Apr 15 11:05:07 2020 -0600
Debug TM, TO, TO Perl, or TR CDN-in-a-Box containers (#4375)
* Conditionally disable compiler optimizations
* Do not time out requests, responses, or db connections in CDN-in-a-Box
if we are debugging
* Run traffic_ops_golang and to-enroll in the foreground when possible
* Debug Traffic Ops with Delve when TO_DEBUG_ENABLE is true
* Debug Traffic Monitor with Delve when TM_DEBUG_ENABLE is true
* Debug Traffic Ops Perl with Devel::Camelcadedb when TO_PERL_DEBUG_ENABLE
is true
* Remove trailing comma from CiaB's cdn.conf
* Document and use "request_timeout" config setting first introduced in 02f83d61b9
* Use CMD for trafficmonitor image, not ENTRYPOINT
* If debugging Traffic Ops, keep the trafficops container running after
traffic_ops_golang exits for easy reloading
* Debug Traffic Router with JPDA when TR_DEBUG_ENABLE is true
* Keep `make debug` from building all packages
* Delve no longer support Go 1.11, disabling checking for an unsupported
version for now
* Re-enable checking for an unsupported Go version now that we are using
Go version 1.14
Revert "Delve no longer support Go 1.11, disabling checking for an unsupported"
This reverts commit 0d731396e806e71c8447a528f00a15b103c65967.
* Add example debugging configuration
* Use "attach" instead of "launch" for example launch.json
* If debugging, leave Traffic Monitor container running after
traffic_monitor exits for easy reloading
---
CHANGELOG.md | 1 +
build/functions.sh | 4 +
docs/source/admin/quick_howto/ciab.rst | 5 +
docs/source/admin/traffic_ops.rst | 4 +
docs/source/development/debugging.rst | 310 +++++++++++++++++++++
.../debugging/to_perl_connection_breakpoint.png | Bin 0 -> 33509 bytes
docs/source/development/index.rst | 1 +
.../traffic_monitor/traffic_monitor_api.rst | 2 +
infrastructure/cdn-in-a-box/Makefile | 20 +-
.../optional/docker-compose.debugging.yml | 83 ++++++
.../cdn-in-a-box/traffic_monitor/Dockerfile | 2 +-
.../{Dockerfile => Dockerfile-debug} | 38 +--
infrastructure/cdn-in-a-box/traffic_monitor/run.sh | 26 +-
.../traffic_monitor/traffic_monitor.cfg | 6 +-
.../Dockerfile => traffic_ops/Dockerfile-debug} | 39 +--
.../Dockerfile => traffic_ops/Dockerfile-go-debug} | 40 +--
infrastructure/cdn-in-a-box/traffic_ops/config.sh | 32 ++-
infrastructure/cdn-in-a-box/traffic_ops/run-go.sh | 19 +-
infrastructure/cdn-in-a-box/traffic_ops/run.sh | 16 +-
.../cdn-in-a-box/traffic_ops/trafficops-init.sh | 23 +-
infrastructure/cdn-in-a-box/traffic_router/run.sh | 21 +-
infrastructure/cdn-in-a-box/variables.env | 9 +
.../docker/build/Dockerfile-traffic_monitor | 2 +
pkg | 11 +-
traffic_monitor/build/build_rpm.sh | 7 +-
traffic_ops/app/conf/cdn.conf | 1 +
traffic_ops/build/build_rpm.sh | 14 +-
27 files changed, 609 insertions(+), 127 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 571f8f3..73e55ac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
- Added an API 1.5 endpoint to GET a single or all records for Let's Encrypt DNS challenge
- Added an API 1.5 endpoint to renew certificates
- Added ability to create multiple objects from generic API Create with a single POST.
+- Added debugging functionality to CDN-in-a-Box.
- Traffic Ops Golang Endpoints
- /api/2.0 for all of the most recent route versions
- /api/1.1/cachegroupparameters/{{cachegroupID}}/{{parameterID}} `(DELETE)`
diff --git a/build/functions.sh b/build/functions.sh
index 87511ff..90628b2 100755
--- a/build/functions.sh
+++ b/build/functions.sh
@@ -136,6 +136,10 @@ function buildRpm () {
local rpm="${pre}.$(uname -m).rpm"
local srpm="${pre}.src.rpm"
echo "Building the rpm."
+ if [[ "$DEBUG_BUILD" == true ]]; then
+ echo 'RPM will not strip binaries before packaging.';
+ echo '%__os_install_post %{nil}' >> /etc/rpm/macros; # Do not strip binaries before packaging
+ fi;
cd "$RPMBUILD" && \
rpmbuild --define "_topdir $(pwd)" \
diff --git a/docs/source/admin/quick_howto/ciab.rst b/docs/source/admin/quick_howto/ciab.rst
index c7854d1..c49533c 100644
--- a/docs/source/admin/quick_howto/ciab.rst
+++ b/docs/source/admin/quick_howto/ciab.rst
@@ -433,3 +433,8 @@ There are some *scripted dashboards* can show beautiful charts. You can display
* ``https://<grafanaHost>/dashboard/script/traffic_ops_cachegroup.js?which=``. The query parameter `which` in this particular URL should be the **cachegroup**. Take CIAB as an example, it can be filled in with **CDN_in_a_Box_Edge** or **CDN_in_a_Box_Edge**.
* ``https://<grafanaHost>/dashboard/script/traffic_ops_deliveryservice.js?which=``. The query parameter `which` in this particular URL should be the **xml_id** of the desired Delivery Service.
* ``https://<grafanaHost>/dashboard/script/traffic_ops_server.js?which=``. The query parameter `which` in this particular URL should be the **hostname** (not **FQDN**). It can be filled in with **edge** or **mid** in CIAB.
+
+Debugging
+---------
+
+See :ref:`dev-debugging-ciab`.
diff --git a/docs/source/admin/traffic_ops.rst b/docs/source/admin/traffic_ops.rst
index 9070cd5..ce663b4 100644
--- a/docs/source/admin/traffic_ops.rst
+++ b/docs/source/admin/traffic_ops.rst
@@ -419,6 +419,7 @@ This file deals with the configuration parameters of running Traffic Ops itself.
:proxy_tls_timeout: An optional field that sets the timeout in seconds for TLS handshakes from Traffic Ops to the `Legacy Perl Script`_. If set to zero, there is no timeout. The default if not specified is zero.
:read_header_timeout: An optional timeout in seconds before which Traffic Ops must be able to finish reading the headers of an incoming request or it will drop the connection. If set to zero, there is no timeout. Default if not specified is zero.
:read_timeout: An optional timeout in seconds before which Traffic Ops must be able to finish reading an entire incoming request (including body) or it will drop the connection. If set to zero, there is no timeout. Default if not specified is zero.
+ :request_timeout: An optional timeout in seconds that serves as the maximum time each Traffic Ops middleware can take to execute. If it is exceeded, the text "server timed out" is served in place of a response. If set to :code:`0`, :code:`60` is used instead. Default if not specified is :code:`60`.
:riak_port: An optional field that sets the port on which Traffic Ops will try to contact Traffic Vault for storage and retrieval of sensitive encryption keys.
.. impl-detail:: The name of this field is derived from the current database used in the implementation of Traffic Vault - `Riak KV <https://riak.com/products/riak-kv/index.html>`_.
@@ -429,6 +430,9 @@ This file deals with the configuration parameters of running Traffic Ops itself.
.. warning:: OAuth support in Traffic Ops is still in its infancy, so most users are advised to avoid defining this field without good cause.
:write_timeout: An optional timeout in seconds set on handlers. After reading a request's header, the server will have this long to send back a response. If set to zero, there is no timeout. Default if not specified is zero.
+
+ .. _admin-routing-blacklist:
+
:routing_blacklist: Optional configuration for explicitly routing requests to TO-Perl via ``perl_routes`` (only routes that are hardcoded to be able to bypass to TO-Perl -- not all Go routes can be bypassed to Perl) or explicitly disabling any routes via ``disabled_routes``.
.. versionadded:: 4.0
diff --git a/docs/source/development/debugging.rst b/docs/source/development/debugging.rst
new file mode 100644
index 0000000..953e1ea
--- /dev/null
+++ b/docs/source/development/debugging.rst
@@ -0,0 +1,310 @@
+..
+..
+.. Licensed under the Apache License, Version 2.0 (the "License");
+.. you may not use this file except in compliance with the License.
+.. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+..
+
+.. role:: bash(code)
+ :language: bash
+
+.. _dev-debugging-ciab:
+
+*****************************
+Debugging inside CDN-in-a-Box
+*****************************
+
+Some CDN-in-a-Box components can be used with a debugger to step through lines of code, set breakpoints, see the state of all variables in each scope, etc. at runtime. Components that support debugging:
+
+* `Traffic Monitor`_
+* `Traffic Ops (Go)`_
+* `Traffic Ops Perl`_
+* `Traffic Router`_
+
+Traffic Monitor
+===============
+
+* Navigate to the ``infrastructure/cdn-in-a-box`` directory. Remove the existing RPMs because they contain release Go binaries do not include useful debugging information. Rebuild the RPMs with no optimization, for debugging:
+
+.. code-block:: shell
+ :caption: Remove release RPMs, then build debug RPMs
+
+ make very-clean
+ make debug
+
+* Still in ``infrastructure/cdn-in-a-box``, open ``variables.env`` and set ``TM_DEBUG_ENABLE`` to ``true``.
+
+* Stop CDN-in-a-Box if it is running and remove any existing volumes. Rebuild the ``trafficmonitor`` image without reusing any cached layers to make sure it uses our fresh ``traffic_monitor.rpm``. Then, start CDN-in-a-Box.
+
+.. code-block:: shell
+ :caption: docker-compose command for debugging Traffic Monitor
+
+ alias mydc='docker-compose -f docker-compose.yml -f docker-compose.expose-ports.yml optional/docker-compose.debugging.yml'
+ mydc down -v
+ mydc build --no-cache trafficmonitor
+ mydc up
+
+* Install `an IDE that supports delve <https://github.com/Microsoft/vscode-go/wiki/Debugging-Go-code-using-VS-Code>`_ and create a debugging configuration over port 2344. If you are using VS Code, the configuration should look like this:
+
+.. code-block:: json
+ :caption: VS Code launch.json for debugging Traffic Monitor
+
+ {
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "Traffic Monitor",
+ "type": "go",
+ "request": "attach",
+ "mode": "remote",
+ "port": 2344,
+ "cwd": "${workspaceRoot}/traffic_monitor",
+ "remotePath": "/tmp/go/src/github.com/apache/trafficcontrol/traffic_monitor",
+ }
+ ]
+ }
+
+* Use the debugging configuration you created to start debugging Traffic Monitor. It should connect without first breaking at any line.
+
+For an example of usage, set a breakpoint at `the o.m.RLock() call in ThreadsafeEvents.Get() <https://github.com/apache/trafficcontrol/blob/RELEASE-4.0.0-RC3/traffic_monitor/health/event.go#L69>`_, then visit http://trafficmonitor.infra.ciab.test/publish/EventLog (see :ref:`Traffic Monitor APIs: /publish/EventLog <tm-publish-EventLog>`).
+
+Traffic Ops (Go)
+================
+
+* Navigate to the ``infrastructure/cdn-in-a-box`` directory. Remove the existing RPMs because they contain release Go binaries do not include useful debugging information. Rebuild the RPMs with no optimization, for debugging:
+
+.. code-block:: shell
+ :caption: Remove release RPMs, then build debug RPMs
+
+ make very-clean
+ make debug
+
+* Still in ``infrastructure/cdn-in-a-box``, open ``variables.env`` and set ``TO_DEBUG_ENABLE`` to ``true``.
+
+* Stop CDN-in-a-Box if it is running and remove any existing volumes. Rebuild the ``trafficops-go`` image without reusing any cached layers to make sure it uses our fresh ``traffic_ops.rpm``. Then, start CDN-in-a-Box.
+
+.. code-block:: shell
+ :caption: docker-compose command for debugging Traffic Ops
+
+ alias mydc='docker-compose -f docker-compose.yml -f docker-compose.expose-ports.yml optional/docker-compose.debugging.yml'
+ mydc down -v
+ mydc build --no-cache trafficops
+ mydc up
+
+* Install `an IDE that supports delve <https://github.com/Microsoft/vscode-go/wiki/Debugging-Go-code-using-VS-Code>`_ and create a debugging configuration over port 2345. If you are using VS Code, the configuration should look like this:
+
+.. code-block:: json
+ :caption: VS Code launch.json for debugging Traffic Ops
+
+ {
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "Traffic Ops",
+ "type": "go",
+ "request": "attach",
+ "mode": "remote",
+ "port": 2345,
+ "cwd": "${workspaceRoot}/traffic_ops/traffic_ops_golang",
+ "remotePath": "/tmp/go/src/github.com/apache/trafficcontrol/traffic_ops/traffic_ops_golang",
+ }
+ ]
+ }
+
+* Use the debugging configuration you created to start debugging Traffic Ops. It should connect without first breaking at any line.
+
+For an example of usage, set a breakpoint at `the log.Debugln() call in TOProfile.Read() <https://github.com/apache/trafficcontrol/blob/RELEASE-4.0.0-RC3/traffic_ops/traffic_ops_golang/profile/profiles.go#L129>`_, then visit https://trafficportal.infra.ciab.test/api/1.5/profiles (after logging into :ref:`tp-overview`).
+
+Traffic Ops Perl
+================
+
+.. deprecated:: ATCv4
+
+* Go to ``infrastructure/cdn-in-a-box`` and open ``variables.env``:
+
+ - Set ``TO_PERL_DEBUG_ENABLE`` to ``true``
+ - Set ``TO_PERL_PORT`` to ``80`` (Camelcadedb will recurse indefinitely if you try debugging requests over HTTPS).
+ - Set ``TO_PERL_SCHEME`` to ``http`` (same reason as for ``TO_PERL_PORT``)
+ - Save and close ``variables.env``.
+
+* Go to ``traffic_ops/app`` and install the Perl modules.
+
+.. code-block:: shell
+ :caption: Install the Perl modules locally
+
+ export PERL5LIB=$(pwd)/local/lib/perl5
+ cpanm -l ./local Carton
+ local/bin/carton install
+
+* Install IntelliJ IDEA Community Edition.
+
+* Get the `Camelcade <https://github.com/Camelcade/Perl5-IDEA>`_ plugin.
+
+* Make a new Perl Remote Debugging debug configuration:
+
+ - The remote project root should be ``/opt/``
+ - Change "Connection mode" to "IDE connects to the perl process".
+ - Server host can remain ``localhost`` or be blank.
+ - Keep the server port at 5000.
+
+* Next, start CDN-in-a-Box with exposed ports and debugging containers:
+
+.. code-block:: shell
+ :caption: docker-compose command for debugging Traffic Ops Perl
+
+ alias mydc='docker-compose -f docker-compose.yml -f docker-compose.expose-ports.yml optional/docker-compose.debugging.yml'
+ mydc down -v
+ mydc up --build
+
+* Watch the ``trafficops-perl`` container so you know when Perl is listening for the IDE connection:
+
+.. code-block:: shell
+ :caption: Watch the ``trafficops-perl`` container's logs
+
+ mydc logs -f trafficops-perl
+
+* Wait until you see this line in the logs. Alternatively, wait about 1 minute after all of the containers start. If you try to debug too early, just wait a few seconds and try again.
+
+ ``Listening for the IDE connection at 0.0.0.0:5000...``
+
+* Start debugging using the configuration you created. If everything works so far, your IDE should highlight the line that execution is halted at within ``morbo``, the Perl web server we are using for debugging (see screenshot).
+
+.. figure:: debugging/to_perl_connection_breakpoint.png
+ :align: center
+ :width: 70%
+ :alt: Debugging connection breakpoint
+
+ Debugging connection breakpoint
+
+* Resume/continue execution by pressing F9 or by clicking the "play" button in the debugging panel. At this point, you can set more breakpoints. **Camelcadedb has the limitation that new breakpoints are only recognized if you set them while the debugger is at a breakpoint or before the IDE connects.**
+
+For an example of usage, set a breakpoint at `the $self->success() call at the end of API::Cdn->index() <https://github.com/apache/trafficcontrol/blob/RELEASE-4.0.0-RC3/traffic_ops/app/lib/API/Cdn.pm#L47>`_, restart CDN-in-a-Box, and use the Python client for Traffic Ops (which is installed in the ``cache`` Docker image) to hit the :ref:`to-api-cdns` endpoint:
+
+.. code-block:: shell
+ :caption: Authenticates and GETs http://trafficops-perl/api/1.3/cdns
+
+ docker-compose exec mid toget --to-user=admin --to-password=twelve --to-url=http://trafficops-perl cdns
+
+Use the :ref:`routing blacklist feature <admin-routing-blacklist>` to route from Traffic Ops to Traffic Ops Perl, which will yield more consistent results.
+
+Traffic Router
+==============
+
+* Navigate to the ``infrastructure/cdn-in-a-box`` directory.
+
+* In ``variables.env``, set ``TR_DEBUG_ENABLE`` to ``true``.
+
+* Install a debugging-capabe Java IDE or text editor of your choice. If unsure, install IntelliJ IDEA Community Edition.
+
+* At the base of the repository (not in the ``cdn-in-a-box`` directory), open the ``traffic_router`` directory in your IDE.
+
+* Add a new "Remote" (Java) debug configuration. Use port 5005.
+
+* Start CDN-in-a-Box, including the "expose ports" "debugging" compose files:
+
+.. code-block:: shell
+ :caption: docker-compose command for debugging Traffic Router
+
+ alias mydc='docker-compose -f docker-compose.yml -f docker-compose.expose-ports.yml optional/docker-compose.debugging.yml'
+ mydc down -v
+ mydc build trafficrouter
+ mydc up -d
+ mydc logs --follow trafficrouter
+
+* Watch the ``trafficrouter`` container's log. After DNS and certificate operations, the enroller, and Traffic Monitor, Traffic Router will start. Look for ``Listening for transport dt_socket at address: 5005`` in the example log below:
+
+.. code-block:: shell
+ :caption: Log of the Docker container for Traffic Router
+
+ Warning:
+ The JKS keystore uses a proprietary format. It is recommended to migrate to PKCS12 which is an industry standard format using "keytool -importkeystore -srckeystore /opt/traffic_router/conf/keyStore.jks -destkeystore /opt/traffic_router/conf/keyStore.jks -deststoretype pkcs12".
+ Certificate stored in file <trafficrouter.infra.ciab.test.crt>
+
+ Warning:
+ The JKS keystore uses a proprietary format. It is recommended to migrate to PKCS12 which is an industry standard format using "keytool -importkeystore -srckeystore /opt/traffic_router/conf/keyStore.jks -destkeystore /opt/traffic_router/conf/keyStore.jks -deststoretype pkcs12".
+ Waiting for enroller initial data load to complete....
+ Waiting for enroller initial data load to complete....
+ Waiting for enroller initial data load to complete....
+ Waiting for enroller initial data load to complete....
+ Waiting for enroller initial data load to complete....
+ Waiting for enroller initial data load to complete....
+ Waiting for enroller initial data load to complete....
+ Waiting for enroller initial data load to complete....
+ Waiting for enroller initial data load to complete....
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ Waiting for Traffic Monitor to start...
+ tail: cannot open '/opt/tomcat/logs/catalina.log' for reading: No such file or directory
+ tail: cannot open '/opt/tomcat/logs/catalina.2020-02-21.log' for reading: No such file or directory
+ ==> /opt/traffic_router/var/log/traffic_router.log <==
+
+ ==> /opt/traffic_router/var/log/access.log <==
+ Tomcat started.
+ tail: '/opt/tomcat/logs/catalina.log' has appeared; following end of new file
+ tail: '/opt/tomcat/logs/catalina.2020-02-21.log' has appeared; following end of new file
+
+ ==> /opt/traffic_router/var/log/traffic_router.log <==
+ INFO 2020-02-21T05:16:07.557 [Thread-3] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidPoller - Waiting for state from mbean path traffic-router:name=languidState
+ INFO 2020-02-21T05:16:07.557 [Thread-4] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidPoller - Waiting for state from mbean path traffic-router:name=languidState
+ INFO 2020-02-21T05:16:07.558 [Thread-5] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidPoller - Waiting for state from mbean path traffic-router:name=languidState
+ INFO 2020-02-21T05:16:07.559 [Thread-6] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidPoller - Waiting for state from mbean path traffic-router:name=languidState
+
+ ==> /opt/tomcat/logs/catalina.log <==
+ Listening for transport dt_socket at address: 5005
+
+ Watch for the line that mentions port 5005 -----------^^^^
+
+ ==> /opt/tomcat/logs/catalina.2020-02-21.log <==
+ 21-Feb-2020 05:16:07.359 WARNING [main] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidNioProtocol.<clinit> Adding BouncyCastle provider
+ 21-Feb-2020 05:16:07.452 WARNING [main] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidNioProtocol.<init> Serving wildcard certs for multiple domains
+ 21-Feb-2020 05:16:07.459 WARNING [main] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidNioProtocol.<init> Serving wildcard certs for multiple domains
+ 21-Feb-2020 05:16:07.459 WARNING [main] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidNioProtocol.<init> Serving wildcard certs for multiple domains
+ 21-Feb-2020 05:16:07.461 INFO [main] com.comcast.cdn.traffic_control.traffic_router.protocol.LanguidNioProtocol.setSslImplementationName setSslImplementation: com.comcast.cdn.traffic_control.traffic_router.protocol.RouterSslImplementation
+
+* When you see that Tomcat is listening for debugger connections on port 5005, start debugging using the debug configuration that you created.
+
+Troubleshooting
+===============
+
+* If you are debugging a Golang project and you don't see the values of all variables, or stepping to the next line puts you several lines ahead, rebuild the Docker image with an RPM built using :bash:`make debug`.
+
+* If you are trying to debug Traffic Ops Perl but the page hangs and you see this warning in the ``trafficops-perl`` container's log:
+
+.. code-block:: shell
+ :caption: Infinite recursion problem with Devel::Camelcadedb when trying to debug a page served over HTTPS
+
+ Deep recursion on subroutine "DB::_get_reference_descriptor" at /usr/local/share/perl5/Devel/Camelcadedb.pm line 584.
+
+
+Make sure Traffic Ops Perl is serving pages over HTTP, not HTTPS.
diff --git a/docs/source/development/debugging/to_perl_connection_breakpoint.png b/docs/source/development/debugging/to_perl_connection_breakpoint.png
new file mode 100644
index 0000000..6144bc8
Binary files /dev/null and b/docs/source/development/debugging/to_perl_connection_breakpoint.png differ
diff --git a/docs/source/development/index.rst b/docs/source/development/index.rst
index b3a9721..bc83f4c 100644
--- a/docs/source/development/index.rst
+++ b/docs/source/development/index.rst
@@ -28,4 +28,5 @@ Use this guide to start developing applications that consume the Traffic Control
traffic_router
traffic_monitor
traffic_stats
+ debugging
documentation_guidelines
diff --git a/docs/source/development/traffic_monitor/traffic_monitor_api.rst b/docs/source/development/traffic_monitor/traffic_monitor_api.rst
index 16e6f50..46a1067 100644
--- a/docs/source/development/traffic_monitor/traffic_monitor_api.rst
+++ b/docs/source/development/traffic_monitor/traffic_monitor_api.rst
@@ -22,6 +22,8 @@ The Traffic Monitor URLs below allow certain query parameters for use in control
.. note:: Unlike :ref:`Traffic Ops API endpoints <to-api>`\ , no authentication is required for any of these, and as such there can be no special role requirements for a user.
+.. _tm-publish-EventLog:
+
``/publish/EventLog``
=====================
Gets a log of recent changes in the availability of polled caches.
diff --git a/infrastructure/cdn-in-a-box/Makefile b/infrastructure/cdn-in-a-box/Makefile
index 24b47f1..c4f2ed4 100644
--- a/infrastructure/cdn-in-a-box/Makefile
+++ b/infrastructure/cdn-in-a-box/Makefile
@@ -29,6 +29,7 @@ ifneq ($(PWD),$(makefile_dir))
$(error This makefile MUST be run from within its directory)
endif
+PKG_FLAGS := -v
BUILD_NUMBER := $(shell git rev-list HEAD 2>/dev/null | wc -l | tr -d '[[:space:]]').$(shell git rev-parse --short=8 HEAD)
TC_VERSION := $(shell cat "../../VERSION")
TOMCAT_VERSION := $(shell grep 'export TOMCAT_VERSION=' ../../traffic_router/build/build_rpm.sh | cut -d '=' -f 2)
@@ -47,11 +48,18 @@ TP_SOURCE := $(wildcard ../../traffic_portal/**/*)
TR_SOURCE := $(wildcard ../../traffic_router/**/*)
TS_SOURCE := $(wildcard ../../traffic_stats/**/*)
-.PHONY: clean very-clean all nearly-all
+.PHONY: clean very-clean all nearly-all debug
# Default target; builds all pre-requisite rpms from source trees
all: traffic_monitor/traffic_monitor.rpm traffic_portal/traffic_portal.rpm traffic_ops/traffic_ops.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm traffic_stats/traffic_stats.rpm
+debug: PKG_FLAGS += -d
+ifneq ($(MAKECMDGOALS), debug)
+debug: $(MAKECMDGOALS)
+else
+debug: all
+endif
+
# Actual output rpm recipies
traffic_monitor/traffic_monitor.rpm: ../../dist/traffic_monitor-$(SPECIAL_SAUCE)
cp -f $? $@
@@ -68,19 +76,19 @@ traffic_stats/traffic_stats.rpm: ../../dist/traffic_stats-$(SPECIAL_SAUCE)
# Dist rpms
../../dist/traffic_monitor-$(SPECIAL_SAUCE): $(TM_SOURCE)
- sudo ../../pkg -v traffic_monitor_build
+ sudo ../../pkg $(PKG_FLAGS) traffic_monitor_build
../../dist/traffic_ops-$(SPECIAL_SAUCE): $(TO_SOURCE)
- sudo ../../pkg -v traffic_ops_build
+ sudo ../../pkg $(PKG_FLAGS) traffic_ops_build
../../dist/traffic_portal-$(SPECIAL_SAUCE): $(TP_SOURCE)
- sudo ../../pkg -v traffic_portal_build
+ sudo ../../pkg $(PKG_FLAGS) traffic_portal_build
../../dist/traffic_rou%er-$(SPECIAL_SAUCE) ../../dist/tomca%-$(SPECIAL_SEASONING): $(TR_SOURCE)
- sudo ../../pkg -v traffic_router_build
+ sudo ../../pkg $(PKG_FLAGS) traffic_router_build
../../dist/traffic_stats-$(SPECIAL_SAUCE): $(TS_SOURCE)
- sudo ../../pkg -v traffic_stats_build
+ sudo ../../pkg $(PKG_FLAGS) traffic_stats_build
clean:
$(RM) traffic_monitor/traffic_monitor.rpm traffic_ops/traffic_ops.rpm traffic_portal/traffic_portal.rpm traffic_router/traffic_router.rpm traffic_router/tomcat.rpm edge/traffic_ops_ort.rpm mid/traffic_ops_ort.rpm traffic_stats/traffic_stats.rpm
diff --git a/infrastructure/cdn-in-a-box/optional/docker-compose.debugging.yml b/infrastructure/cdn-in-a-box/optional/docker-compose.debugging.yml
new file mode 100644
index 0000000..9e14dba
--- /dev/null
+++ b/infrastructure/cdn-in-a-box/optional/docker-compose.debugging.yml
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Exposes debugging ports
+
+---
+version: '2.1'
+
+services:
+ trafficmonitor:
+ build:
+ context: .
+ dockerfile: traffic_monitor/Dockerfile-debug
+ image: trafficmonitor-debug
+ ports:
+ - "2344:2344" #Delve debugging port
+ depends_on:
+ - trafficmonitor-nondebug
+ trafficops:
+ build:
+ context: .
+ dockerfile: traffic_ops/Dockerfile-go-debug
+ image: trafficops-go-debug
+ ports:
+ - "2345:2345" #Delve debugging port
+ depends_on:
+ - trafficops-go-nondebug
+ trafficops-perl:
+ build:
+ context: .
+ dockerfile: traffic_ops/Dockerfile-debug
+ image: trafficops-perl-debug
+ ports:
+ - "5000:5000" #Camelcade debugging port
+ depends_on:
+ - trafficops-perl-nondebug
+ trafficrouter:
+ ports:
+ - "5005:5005" # JPDA debugging port
+ # The trafficmonitor-nondebug service exists to ensure that the trafficmonitor
+ # base image exists before building trafficmonitor-debug.
+ trafficmonitor-nondebug:
+ image: trafficmonitor
+ build:
+ context: .
+ dockerfile: traffic_monitor/Dockerfile
+ args:
+ TRAFFIC_MONITOR_RPM: traffic_monitor/traffic_monitor.rpm
+ command: /usr/bin/true
+ # The trafficops-go-nondebug service ensures that the trafficops-go base image
+ # exists before trying to build trafficops-go-debug.
+ trafficops-go-nondebug:
+ image: trafficops-go
+ build:
+ context: ../..
+ dockerfile: infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-go
+ args:
+ TRAFFIC_OPS_RPM: infrastructure/cdn-in-a-box/traffic_ops/traffic_ops.rpm
+ command: /usr/bin/true
+ # The trafficops-perl-nondebug service exists to ensure that the trafficops-perl
+ # base image exists before building trafficmonitor-debug.
+ trafficops-perl-nondebug:
+ image: trafficops-perl
+ build:
+ context: ../..
+ dockerfile: infrastructure/cdn-in-a-box/traffic_ops/Dockerfile
+ args:
+ TRAFFIC_OPS_RPM: infrastructure/cdn-in-a-box/traffic_ops/traffic_ops.rpm
+ command: /usr/bin/true
diff --git a/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile b/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
index 9cf40d7..2e63309 100644
--- a/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
+++ b/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
@@ -48,4 +48,4 @@ ADD enroller/server_template.json \
EXPOSE 80
ADD traffic_monitor/run.sh /
-ENTRYPOINT /run.sh
+CMD /run.sh
diff --git a/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile b/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile-debug
similarity index 54%
copy from infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
copy to infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile-debug
index 9cf40d7..eba37e3 100644
--- a/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
+++ b/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile-debug
@@ -14,38 +14,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
############################################################
-# Dockerfile to build Traffic Monitor container images
+# Dockerfile to build Traffic Monitor debugging container image
# Based on CentOS
############################################################
-FROM centos/systemd
-
-# Default values for RPM -- override with `docker build --build-arg RPM=...'
-ARG RPM=traffic_monitor/traffic_monitor.rpm
-ADD $RPM /
-
-RUN yum install -y epel-release && \
- yum install -y \
- jq \
- nmap-ncat \
- iproute \
- net-tools \
- gettext \
- bind-utils \
- openssl \
- initscripts && \
- yum install -y /$(basename $RPM) && \
- rm /$(basename $RPM) && \
- yum clean all
-
-RUN mkdir -p /opt/traffic_monitor/conf
-ADD traffic_monitor/traffic_monitor.cfg /opt/traffic_monitor/conf/traffic_monitor.cfg.template
-
-ADD enroller/server_template.json \
- traffic_ops/to-access.sh \
- /
+FROM centos/systemd as build-delve
+RUN yum -y install epel-release && \
+ yum -y install golang && \
+ go get -u github.com/go-delve/delve/cmd/dlv
-EXPOSE 80
-ADD traffic_monitor/run.sh /
-ENTRYPOINT /run.sh
+FROM trafficmonitor
+COPY --from=build-delve /root/go/bin /usr/bin
diff --git a/infrastructure/cdn-in-a-box/traffic_monitor/run.sh b/infrastructure/cdn-in-a-box/traffic_monitor/run.sh
index d2cd9b1..59e24ce 100755
--- a/infrastructure/cdn-in-a-box/traffic_monitor/run.sh
+++ b/infrastructure/cdn-in-a-box/traffic_monitor/run.sh
@@ -107,6 +107,28 @@ until [ $(to-get "/api/2.0/cdns/$CDN_NAME/snapshot" 2>/dev/null | jq -c -e '.res
sleep 3;
done
+if [[ "$TM_DEBUG_ENABLE" == true ]]; then
+ day_in_ms=$(( 1000 * 60 * 60 * 24 )); # Timing out debugging after 1 day seems fair
+ set -o allexport;
+ HTTP_TIMEOUT_MS=$day_in_ms
+ SERVER_READ_TIMEOUT_MS=$day_in_ms
+ SERVER_WRITE_TIMEOUT_MS=$day_in_ms
+ set +o allexport;
+else
+ set -o allexport;
+ HTTP_TIMEOUT_MS=2000
+ SERVER_READ_TIMEOUT_MS=10000
+ SERVER_WRITE_TIMEOUT_MS=10000
+ set +o allexport;
+fi;
+
envsubst < /opt/traffic_monitor/conf/traffic_monitor.cfg.template > /opt/traffic_monitor/conf/traffic_monitor.cfg
-cd /opt/traffic_monitor
-/opt/traffic_monitor/bin/traffic_monitor -opsCfg /opt/traffic_monitor/conf/traffic_ops.cfg -config /opt/traffic_monitor/conf/traffic_monitor.cfg
+
+traffic_monitor_command=(/opt/traffic_monitor/bin/traffic_monitor -opsCfg /opt/traffic_monitor/conf/traffic_ops.cfg -config /opt/traffic_monitor/conf/traffic_monitor.cfg);
+if [[ "$TM_DEBUG_ENABLE" == true ]]; then
+ dlv '--continue' '--listen=:2344' '--accept-multiclient=true' '--headless=true' '--api-version=2' exec \
+ "${traffic_monitor_command[0]}" -- "${traffic_monitor_command[@]:1}" &
+ tail -f /dev/null;
+else
+ "${traffic_monitor_command[@]}"
+fi;
diff --git a/infrastructure/cdn-in-a-box/traffic_monitor/traffic_monitor.cfg b/infrastructure/cdn-in-a-box/traffic_monitor/traffic_monitor.cfg
index 8d98cec..4b87440 100644
--- a/infrastructure/cdn-in-a-box/traffic_monitor/traffic_monitor.cfg
+++ b/infrastructure/cdn-in-a-box/traffic_monitor/traffic_monitor.cfg
@@ -2,7 +2,7 @@
"cache_health_polling_interval_ms": 6000,
"cache_stat_polling_interval_ms": 6000,
"monitor_config_polling_interval_ms": 15000,
- "http_timeout_ms": 2000,
+ "http_timeout_ms": $HTTP_TIMEOUT_MS,
"peer_polling_interval_ms": 5000,
"peer_optimistic": true,
"peer_optimisitc_quorum_min": 0,
@@ -16,8 +16,8 @@
"log_location_warning": "$TM_LOG_WARNING",
"log_location_info": "$TM_LOG_INFO",
"log_location_debug": "$TM_LOG_DEBUG",
- "serve_read_timeout_ms": 10000,
- "serve_write_timeout_ms": 10000,
+ "serve_read_timeout_ms": $SERVER_READ_TIMEOUT_MS,
+ "serve_write_timeout_ms": $SERVER_WRITE_TIMEOUT_MS,
"http_poll_no_sleep": false,
"static_file_dir": "/opt/traffic_monitor/static/"
}
diff --git a/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-debug
similarity index 53%
copy from infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
copy to infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-debug
index 9cf40d7..e39d87b 100644
--- a/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
+++ b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-debug
@@ -14,38 +14,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
############################################################
-# Dockerfile to build Traffic Monitor container images
-# Based on CentOS
+# Dockerfile to build Traffic Ops container images
+# Based on CentOS 7.2
############################################################
-FROM centos/systemd
-
-# Default values for RPM -- override with `docker build --build-arg RPM=...'
-ARG RPM=traffic_monitor/traffic_monitor.rpm
-ADD $RPM /
-
-RUN yum install -y epel-release && \
- yum install -y \
- jq \
- nmap-ncat \
- iproute \
- net-tools \
- gettext \
- bind-utils \
- openssl \
- initscripts && \
- yum install -y /$(basename $RPM) && \
- rm /$(basename $RPM) && \
- yum clean all
-
-RUN mkdir -p /opt/traffic_monitor/conf
-ADD traffic_monitor/traffic_monitor.cfg /opt/traffic_monitor/conf/traffic_monitor.cfg.template
-
-ADD enroller/server_template.json \
- traffic_ops/to-access.sh \
- /
+FROM trafficops-perl
-EXPOSE 80
-ADD traffic_monitor/run.sh /
-ENTRYPOINT /run.sh
+RUN cpanm \
+ Test::More \
+ Devel::Camelcadedb && \
+ rm -r /root/.cpanm
diff --git a/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-go-debug
similarity index 53%
copy from infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
copy to infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-go-debug
index 9cf40d7..9f9d81d 100644
--- a/infrastructure/cdn-in-a-box/traffic_monitor/Dockerfile
+++ b/infrastructure/cdn-in-a-box/traffic_ops/Dockerfile-go-debug
@@ -14,38 +14,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
############################################################
-# Dockerfile to build Traffic Monitor container images
-# Based on CentOS
+# Dockerfile to build Traffic Ops debugging container image
+# Based on CentOS 7.2
############################################################
-FROM centos/systemd
-
-# Default values for RPM -- override with `docker build --build-arg RPM=...'
-ARG RPM=traffic_monitor/traffic_monitor.rpm
-ADD $RPM /
-
-RUN yum install -y epel-release && \
- yum install -y \
- jq \
- nmap-ncat \
- iproute \
- net-tools \
- gettext \
- bind-utils \
- openssl \
- initscripts && \
- yum install -y /$(basename $RPM) && \
- rm /$(basename $RPM) && \
- yum clean all
-
-RUN mkdir -p /opt/traffic_monitor/conf
-ADD traffic_monitor/traffic_monitor.cfg /opt/traffic_monitor/conf/traffic_monitor.cfg.template
-
-ADD enroller/server_template.json \
- traffic_ops/to-access.sh \
- /
+FROM centos:7 as get-delve
+RUN yum -y install epel-release && \
+ yum -y install golang && \
+ go get -u github.com/go-delve/delve/cmd/dlv
-EXPOSE 80
-ADD traffic_monitor/run.sh /
-ENTRYPOINT /run.sh
+FROM trafficops-go
+COPY --from=get-delve /root/go/bin /usr/bin
diff --git a/infrastructure/cdn-in-a-box/traffic_ops/config.sh b/infrastructure/cdn-in-a-box/traffic_ops/config.sh
index 4a0093a..2137372 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops/config.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops/config.sh
@@ -36,7 +36,7 @@
# TP_HOST
#
# Check that env vars are set
-envvars=( DB_SERVER DB_PORT DB_ROOT_PASS DB_USER DB_USER_PASS ADMIN_USER ADMIN_PASS DOMAIN TO_PERL_HOST TO_PERL_PORT TO_HOST TO_PORT TP_HOST)
+envvars=( DB_SERVER DB_PORT DB_ROOT_PASS DB_USER DB_USER_PASS ADMIN_USER ADMIN_PASS DOMAIN TO_PERL_HOST TO_PERL_PORT TO_PERL_SCHEME TO_HOST TO_PORT TP_HOST)
for v in $envvars
do
if [[ -z $$v ]]; then echo "$v is unset"; exit 1; fi
@@ -67,11 +67,15 @@ key="$X509_INFRA_KEY_FILE"
echo "crt=$crt"
echo "key=$key"
+if [[ "$TO_DEBUG_ENABLE" == true ]]; then
+ DEBUGGING_TIMEOUT=$(( 60 * 60 * 24 )); # Timing out debugging after 1 day seems fair
+fi;
+
cat <<-EOF >/opt/traffic_ops/app/conf/cdn.conf
{
"hypnotoad" : {
"listen" : [
- "https://$TO_PERL_FQDN:$TO_PERL_PORT?cert=$crt&key=$key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED"
+ "$TO_PERL_SCHEME://$TO_PERL_FQDN:$TO_PERL_PORT?cert=$crt&key=$key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:!RC4:!MD5:!aNULL:!EDH:!ED"
],
"user" : "trafops",
"group" : "trafops",
@@ -82,25 +86,33 @@ cat <<-EOF >/opt/traffic_ops/app/conf/cdn.conf
"traffic_ops_golang" : {
"insecure": true,
"port" : "$TO_PORT",
- "proxy_timeout" : 60,
+ "proxy_timeout" : ${DEBUGGING_TIMEOUT:-60},
"proxy_keep_alive" : 60,
- "proxy_tls_timeout" : 60,
- "proxy_read_header_timeout" : 60,
- "read_timeout" : 60,
- "read_header_timeout" : 60,
- "write_timeout" : 60,
- "idle_timeout" : 60,
+ "proxy_tls_timeout" : ${DEBUGGING_TIMEOUT:-60},
+ "proxy_read_header_timeout" : ${DEBUGGING_TIMEOUT:-60},
+ "read_timeout" : ${DEBUGGING_TIMEOUT:-60},
+ "read_header_timeout" : ${DEBUGGING_TIMEOUT:-60},
+ "request_timeout" : ${DEBUGGING_TIMEOUT:-60},
+ "write_timeout" : ${DEBUGGING_TIMEOUT:-60},
+ "idle_timeout" : ${DEBUGGING_TIMEOUT:-60},
"log_location_error": "$TO_LOG_ERROR",
"log_location_warning": "$TO_LOG_WARNING",
"log_location_info": "$TO_LOG_INFO",
"log_location_debug": "$TO_LOG_DEBUG",
"log_location_event": "$TO_LOG_EVENT",
"max_db_connections": 20,
+ "db_conn_max_lifetime_seconds": ${DEBUGGING_TIMEOUT:-60},
+ "db_query_timeout_seconds": ${DEBUGGING_TIMEOUT:-20},
"backend_max_connections": {
"mojolicious": 4
},
"whitelisted_oauth_urls": [],
- "oauth_client_secret": ""
+ "oauth_client_secret": "",
+ "routing_blacklist": {
+ "ignore_unknown_routes": false,
+ "perl_routes": [],
+ "disabled_routes": []
+ }
},
"cors" : {
"access_control_allow_origin" : "*"
diff --git a/infrastructure/cdn-in-a-box/traffic_ops/run-go.sh b/infrastructure/cdn-in-a-box/traffic_ops/run-go.sh
index c4f77ad..867c4f7 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops/run-go.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops/run-go.sh
@@ -35,7 +35,10 @@
# Check that env vars are set
-set -x
+# Setting the monitor shell option enables job control, which we need in order
+# to bring traffic_ops_golang back to the foreground.
+set -o xtrace -o monitor;
+
envvars=( DB_SERVER DB_PORT DB_ROOT_PASS DB_USER DB_USER_PASS ADMIN_USER ADMIN_PASS)
for v in $envvars
do
@@ -67,7 +70,12 @@ touch /var/log/traffic_ops/traffic_ops.log
# enroll in the background so traffic_ops_golang can run in foreground
TO_USER=$TO_ADMIN_USER TO_PASSWORD=$TO_ADMIN_PASSWORD to-enroll $(hostname -s) &
-./bin/traffic_ops_golang -cfg $CDNCONF -dbcfg $DBCONF -riakcfg $RIAKCONF &
+traffic_ops_golang_command=(./bin/traffic_ops_golang -cfg "$CDNCONF" -dbcfg "$DBCONF" -riakcfg "$RIAKCONF");
+if [[ "$TO_DEBUG_ENABLE" == true ]]; then
+ traffic_ops_golang_command=(dlv '--accept-multiclient' '--continue' '--listen=:2345' '--headless=true' '--api-version=2' exec
+ "${traffic_ops_golang_command[0]}" -- "${traffic_ops_golang_command[@]:1}");
+fi;
+"${traffic_ops_golang_command[@]}" &
to-enroll "to" ALL || (while true; do echo "enroll failed."; sleep 3 ; done)
@@ -101,4 +109,9 @@ if [[ "$AUTO_SNAPQUEUE_ENABLED" = true ]]; then
to-auto-snapqueue $AUTO_SNAPQUEUE_SERVERS $CDN_NAME
fi
-exec tail -f /dev/null
+fg '"${traffic_ops_golang_command[@]}"'; # Bring traffic_ops_golang to foreground
+fg; # Bring to-enroll to foreground if it is still running
+
+if [[ "$TO_DEBUG_ENABLE" == true ]]; then
+ tail -f /dev/null;
+fi;
diff --git a/infrastructure/cdn-in-a-box/traffic_ops/run.sh b/infrastructure/cdn-in-a-box/traffic_ops/run.sh
index 56532f0..1af0666 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops/run.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops/run.sh
@@ -114,7 +114,21 @@ cd $TO_DIR && \
# Add admin user -- all other users should be created using API
/adduser.pl $TO_ADMIN_USER $TO_ADMIN_PASSWORD admin | psql -v ON_ERROR_STOP=1 -U$DB_USER -h$DB_SERVER $DB_NAME || { echo "adding traffic_ops admin user failed!"; exit 1; }
-cd $TO_DIR && $TO_DIR/local/bin/hypnotoad script/cdn
+cd $TO_DIR || exit 1;
+
+if [[ "$TO_PERL_DEBUG_ENABLE" == true ]]; then
+ set -o allexport;
+ PERL5_DEBUG_HOST=0.0.0.0;
+ PERL5_DEBUG_PORT=5000;
+ PERL5_DEBUG_ROLE=server;
+ MOJO_LISTEN="${TO_PERL_SCHEME}://*:${TO_PERL_PORT}?cert=${X509_CA_DIR}/${INFRA_FQDN}.crt&key=${X509_CA_DIR}/${INFRA_FQDN}.key&verify=0x00&ciphers=AES128-GCM-SHA256:HIGH:"'!RC4:!MD5:!aNULL:!EDH:!ED';
+ MOJO_INACTIVITY_TIMEOUT=$(( 60 * 60 * 24 )); # 24 hours
+ set +o allexport;
+
+ perl -d:Camelcadedb $TO_DIR/local/bin/morbo script/cdn &
+else
+ $TO_DIR/local/bin/hypnotoad script/cdn &
+fi;
until [[ -f ${ENROLLER_DIR}/enroller-started ]]; do
echo "waiting for enroller"
diff --git a/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh b/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh
index ac7b099..6463391 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh
@@ -98,5 +98,26 @@ load_data_from() {
cd -
}
-# First, load required data at the top level
+# If Traffic Router debugging is enabled, keep zone generation from timing out
+# (for 5 minutes), in case that is what you are debugging
+traffic_router_zonemanager_timeout() {
+ if [[ "$TR_DEBUG_ENABLE" != true ]]; then
+ return;
+ fi;
+
+ local modified_crconfig crconfig_path zonemanager_timeout;
+ crconfig_path=/traffic_ops_data/profiles/040-CCR_CIAB.json;
+ modified_crconfig="$(mktemp)";
+ # 5 minutes, which is the default zonemanager.cache.maintenance.interval value
+ zonemanager_timeout="$(( 60 * 5 ))";
+ jq \
+ --arg zonemanager_timeout $zonemanager_timeout \
+ '.params = .params + [{"configFile": "CRConfig.json", "name": "zonemanager.init.timeout", "value": $zonemanager_timeout}]' \
+ <$crconfig_path >"$modified_crconfig";
+ mv "$modified_crconfig" $crconfig_path;
+}
+
+traffic_router_zonemanager_timeout
+
+# Load required data at the top level
load_data_from /traffic_ops_data
diff --git a/infrastructure/cdn-in-a-box/traffic_router/run.sh b/infrastructure/cdn-in-a-box/traffic_router/run.sh
index 5c2f63e..bd2135f 100755
--- a/infrastructure/cdn-in-a-box/traffic_router/run.sh
+++ b/infrastructure/cdn-in-a-box/traffic_router/run.sh
@@ -17,6 +17,16 @@
# under the License.
NAME="Traffic Router Application"
+function longer_dns_timeout() {
+ local day_in_ms dns_properties;
+ day_in_ms=$(( 1000 * 60 * 60 * 24 )); # Timing out debugging after 1 day seems fair
+ dns_properties=/opt/traffic_router/conf/dns.properties;
+ <<-DNS_CONFIG_LINES cat >> $dns_properties;
+ dns.tcp.timeout.task=$(( day_in_ms ))
+ dns.udp.timeout.task=$(( day_in_ms ))
+DNS_CONFIG_LINES
+}
+
set-dns.sh
insert-self-into-dns.sh
@@ -38,6 +48,11 @@ CATALINA_OPTS="\
-XX:+UnlockExperimentalVMOptions \
-XX:InitiatingHeapOccupancyPercent=30"
+if [[ "$TR_DEBUG_ENABLE" == true ]]; then
+ export JPDA_OPTS="-agentlib:jdwp=transport=dt_socket,address=5005,server=y,suspend=n";
+ longer_dns_timeout;
+fi;
+
JAVA_HOME=/opt/java
JAVA_OPTS="\
-Djava.library.path=/usr/lib64 \
@@ -113,6 +128,10 @@ until nc $TM_FQDN $TM_PORT </dev/null >/dev/null 2>&1; do
done
touch $LOGFILE $ACCESSLOG
-exec /opt/tomcat/bin/catalina.sh run &
+if [[ "$TR_DEBUG_ENABLE" == true ]]; then
+ exec /opt/tomcat/bin/catalina.sh jpda start &
+else
+ exec /opt/tomcat/bin/catalina.sh run &
+fi;
tail -F $CATALINA_OUT $CATALINA_LOG $LOGFILE $ACCESSLOG
diff --git a/infrastructure/cdn-in-a-box/variables.env b/infrastructure/cdn-in-a-box/variables.env
index c5140b2..01f8e73 100644
--- a/infrastructure/cdn-in-a-box/variables.env
+++ b/infrastructure/cdn-in-a-box/variables.env
@@ -68,11 +68,20 @@ TM_LOG_INFO=stdout
TM_LOG_DEBUG=stdout
TO_ADMIN_PASSWORD=twelve
TO_ADMIN_USER=admin
+# Set TM_DEBUG_ENABLE to true`to debug Traffic Monitor with Delve
+TM_DEBUG_ENABLE=false
+# Set TO_DEBUG_ENABLE to true`to debug Traffic Ops with Delve
+TO_DEBUG_ENABLE=false
+# Set TO_PERL_DEBUG_ENABLE to true`to debug Traffic Ops Perl with Devel::Camelcadedb
+TO_PERL_DEBUG_ENABLE=false
+# Set TR_DEBUG_ENABLE to true`to debug Traffic Router with with JPDA
+TR_DEBUG_ENABLE=false
TO_EMAIL=cdnadmin@example.com
TO_HOST=trafficops
TO_PORT=443
TO_PERL_HOST=trafficops-perl
TO_PERL_PORT=443
+TO_PERL_SCHEME=https
TO_SECRET=blahblah
TO_LOG_ERROR=stdout
TO_LOG_WARNING=stdout
diff --git a/infrastructure/docker/build/Dockerfile-traffic_monitor b/infrastructure/docker/build/Dockerfile-traffic_monitor
index e404061..b8e854f 100644
--- a/infrastructure/docker/build/Dockerfile-traffic_monitor
+++ b/infrastructure/docker/build/Dockerfile-traffic_monitor
@@ -32,6 +32,8 @@ RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \
### traffic_monitor specific requirements
RUN yum -y install \
+ # gcc is necessary in case -gcflags 'all=-N -l' is passed to go build
+ gcc \
git \
rpm-build && \
yum -y clean all
diff --git a/pkg b/pkg
index 0c3cf04..8a6168f 100755
--- a/pkg
+++ b/pkg
@@ -84,7 +84,7 @@ fi
# Parse command line arguments
verbose=0
-while getopts :lvq? opt; do
+while getopts :qvld? opt; do
case $opt in
\?)
PROJECTS=`$SELF -l | sed "s/^/ - /"`
@@ -92,6 +92,7 @@ while getopts :lvq? opt; do
echo " -q Quiet mode. Supresses output."
echo " -v Verbose mode. Lists all build output."
echo " -l List available projects."
+ echo " -d Disable compiler optimizations for debugging."
echo
echo " If no projects are listed, all projects will be packaged."
echo " Valid projects:"
@@ -108,6 +109,12 @@ while getopts :lvq? opt; do
"${COMPOSECMD[@]}" -f $COMPOSE_FILE config --services
exit $?
;;
+ d)
+ echo '-d is set! Disabling all compiler optimizations for debugging...';
+ # If DEBUG_BUILD is true, then Golang binaries will remain unoptimized and
+ # RPM packaging will not strip binaries.
+ RUN_OPTIONS+=(-e 'DEBUG_BUILD=true');
+ ;;
esac
done
@@ -128,7 +135,7 @@ while (( "$#" )); do
exec >/dev/null 2>&1
fi
"${COMPOSECMD[@]}" -f $COMPOSE_FILE build $1 || exit 1
- "${COMPOSECMD[@]}" -f $COMPOSE_FILE run --rm $1 || exit 1
+ "${COMPOSECMD[@]}" -f $COMPOSE_FILE run "${RUN_OPTIONS[@]}" --rm $1 || exit 1
) || {
# Don't totally bail out, but make note of the failures.
failure=1
diff --git a/traffic_monitor/build/build_rpm.sh b/traffic_monitor/build/build_rpm.sh
index 308ce83..35b438d 100755
--- a/traffic_monitor/build/build_rpm.sh
+++ b/traffic_monitor/build/build_rpm.sh
@@ -48,7 +48,12 @@ function initBuildArea() {
{ echo "Could not get go package dependencies"; exit 1; }
# compile traffic_monitor
- go build -v -ldflags "-X main.GitRevision=`git rev-parse HEAD` -X main.BuildTimestamp=`date +'%Y-%M-%dT%H:%M:%s'` -X main.Version=${TC_VERSION}" || \
+ go_build=(go build -v)
+ if [[ "$DEBUG_BUILD" == true ]]; then
+ echo 'DEBUG_BUILD is enabled, building without optimization or inlining...';
+ go_build+=(-gcflags 'all=-N -l');
+ fi;
+ "${go_build[@]}" -ldflags "-X main.GitRevision=$(git rev-parse HEAD) -X main.BuildTimestamp=$(date +'%Y-%M-%dT%H:%M:%s') -X main.Version=${TC_VERSION}" || \
{ echo "Could not build traffic_monitor binary"; exit 1; }
rsync -av ./ "$tm_dest"/ || \
diff --git a/traffic_ops/app/conf/cdn.conf b/traffic_ops/app/conf/cdn.conf
index f7e27f1..5f68106 100644
--- a/traffic_ops/app/conf/cdn.conf
+++ b/traffic_ops/app/conf/cdn.conf
@@ -16,6 +16,7 @@
"proxy_tls_timeout" : 60,
"proxy_read_header_timeout" : 60,
"read_timeout" : 60,
+ "request_timeout" : 60,
"read_header_timeout" : 60,
"write_timeout" : 60,
"idle_timeout" : 60,
diff --git a/traffic_ops/build/build_rpm.sh b/traffic_ops/build/build_rpm.sh
index 6fe6562..6fa490a 100755
--- a/traffic_ops/build/build_rpm.sh
+++ b/traffic_ops/build/build_rpm.sh
@@ -49,25 +49,30 @@ function initBuildArea() {
# compile traffic_ops_golang
pushd traffic_ops_golang
- go build -v -ldflags "-X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x`git rev-parse HEAD`" || \
+ go_build=(go build -v);
+ if [[ "$DEBUG_BUILD" == true ]]; then
+ echo 'DEBUG_BUILD is enabled, building without optimization or inlining...';
+ go_build+=(-gcflags 'all=-N -l');
+ fi;
+ "${go_build[@]}" -ldflags "-X main.version=traffic_ops-${TC_VERSION}-${BUILD_NUMBER}.${RHEL_VERSION} -B 0x$(git rev-parse HEAD)" || \
{ echo "Could not build traffic_ops_golang binary"; exit 1; }
popd
# compile db/admin
pushd app/db
- go build -v -o admin || \
+ "${go_build[@]}" -o admin || \
{ echo "Could not build db/admin binary"; exit 1; }
popd
# compile TO profile converter
pushd install/bin/convert_profile
- go build -v || \
+ "${go_build[@]}" || \
{ echo "Could not build convert_profile binary"; exit 1; }
popd
# compile atstccfg
pushd ort/atstccfg
- go build -v -ldflags "-X main.GitRevision=`git rev-parse HEAD` -X main.BuildTimestamp=`date +'%Y-%M-%dT%H:%M:%s'` -X main.Version=${TC_VERSION}" || \
+ "${go_build[@]}" -ldflags "-X main.GitRevision=`git rev-parse HEAD` -X main.BuildTimestamp=`date +'%Y-%M-%dT%H:%M:%s'` -X main.Version=${TC_VERSION}" || \
{ echo "Could not build atstccfg binary"; exit 1; }
popd
@@ -91,6 +96,7 @@ function initBuildArea() {
{ echo "Could not create tar archive $to_ort_dest: $?"; exit 1; }
export PLUGINS=$(grep -l -P '(?<!func )AddPlugin\(' ${TO_DIR}/traffic_ops_golang/plugin/*.go | xargs -I '{}' basename {} '.go')
+
echo "The build area has been initialized."
}