You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by da...@apache.org on 2019/02/08 15:32:19 UTC
[trafficcontrol] branch master updated: ORT.py now implements all
the same command line flags as the Perl script (#3283)
This is an automated email from the ASF dual-hosted git repository.
dangogh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new cae82df ORT.py now implements all the same command line flags as the Perl script (#3283)
cae82df is described below
commit cae82dff75ed7d5a93b334961fe84993e703d1c3
Author: ocket8888 <oc...@gmail.com>
AuthorDate: Fri Feb 8 08:32:12 2019 -0700
ORT.py now implements all the same command line flags as the Perl script (#3283)
* ORT.py now implements all the same command line flags as the Perl script
---
infrastructure/cdn-in-a-box/edge/run.sh | 2 +-
.../cdn-in-a-box/ort/traffic_ops_ort.crontab | 2 +-
.../cdn-in-a-box/ort/traffic_ops_ort/__init__.py | 101 +++----
.../ort/traffic_ops_ort/config_files.py | 144 +++++-----
.../ort/traffic_ops_ort/configuration.py | 312 ++++++++++-----------
.../ort/traffic_ops_ort/main_routines.py | 224 +++++++--------
.../cdn-in-a-box/ort/traffic_ops_ort/packaging.py | 25 +-
.../cdn-in-a-box/ort/traffic_ops_ort/services.py | 133 +++++----
.../cdn-in-a-box/ort/traffic_ops_ort/to_api.py | 230 +++++++++------
9 files changed, 594 insertions(+), 579 deletions(-)
diff --git a/infrastructure/cdn-in-a-box/edge/run.sh b/infrastructure/cdn-in-a-box/edge/run.sh
index d0a779d..d88c7a6 100755
--- a/infrastructure/cdn-in-a-box/edge/run.sh
+++ b/infrastructure/cdn-in-a-box/edge/run.sh
@@ -66,7 +66,7 @@ while [[ -z "$(testenrolled)" ]]; do
done
# Wait for SSL keys to exist
-until to-get "api/1.3/cdns/name/$CDN/sslkeys"; do
+until to-get "api/1.3/cdns/name/$CDN/sslkeys" && [[ "$(to-get api/1.3/cdns/name/$CDN/sslkeys)" != '{"response":[]}' ]]; do
echo 'waiting for SSL keys to exist'
sleep 3
done
diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort.crontab b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort.crontab
index ccb522c..7adce29 100644
--- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort.crontab
+++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort.crontab
@@ -14,4 +14,4 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-*/1 * * * * /usr/bin/traffic_ops_ort -k SYNCDS ALL https://$TO_FQDN $TO_ADMIN_USER:$TO_ADMIN_PASSWORD >> /var/log/ort.log 2>> /var/log/ort.log
+*/1 * * * * /usr/bin/traffic_ops_ort -k --dispersion 0 SYNCDS ALL https://$TO_FQDN $TO_ADMIN_USER:$TO_ADMIN_PASSWORD >> /var/log/ort.log 2>> /var/log/ort.log
diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py
index d73aa62..ceb420b 100644
--- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py
+++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/__init__.py
@@ -28,7 +28,7 @@ This package provides an executable script named :program:`traffic_ops_ort`
Usage
=====
-``traffic_ops_ort [-k] [--dispersion DISP] [--login_dispersion DISP] [--retries RETRIES] [--wait_for_parents] [--rev_proxy_disabled] [--ts-root PATH] MODE LOG_LEVEL TO_URL LOGIN``
+``traffic_ops_ort [-k] [--dispersion DISP] [--login_dispersion DISP] [--retries RETRIES] [--wait_for_parents INT] [--rev_proxy_disable] [--ts-root PATH] MODE LOG_LEVEL TO_URL LOGIN``
``traffic_ops_ort [-v]``
@@ -50,37 +50,25 @@ Usage
Wait a random number between 0 and ``DISP`` seconds before starting. (Default: 300)
- .. caution:: This option is not implemented yet; it has no effect and even the default is not
- used.
-
.. option:: --login_dispersion DISP
Wait a random number between 0 and ``DISP`` seconds before authenticating with Traffic Ops.
(Default: 0)
- .. caution:: This option is not implemented yet; it has no effect.
-
.. option:: --retries RETRIES
If connection to Traffic Ops fails, retry ``RETRIES`` times before giving up (Default: 3).
- .. caution:: This option is not implemented yet; it has no effect and even the default is not
- used.
-
-.. option:: --wait_for_parents
+.. option:: --wait_for_parents INT
- Do not apply updates if parents of this server have pending updates.
+ If ``INT`` is anything but 0, do not apply updates if parents of this server have pending
+ updates. This option requires an integer argument for legacy compatibility reasons; 0 is
+ considered ``False``, anything else is ``True``. (Default: 1)
- .. caution:: This option is not implemented yet; it has no effect and currently the default
- behavior is to wait for parents regardless of the presence - or lack thereof - of this option
-
-.. option:: --rev_prox_disabled
+.. option:: --rev_prox_disable
Make requests directly to the Traffic Ops server, bypassing a reverse proxy if one exists.
- .. caution:: This option is not implemented yet; :mod:`traffic_ops_ort` will make requests
- directly to the provided :option:`TO_URL`
-
.. option:: --ts_root PATH
An optional flag which, if present, specifies the absolute path to the install directory of
@@ -161,13 +149,17 @@ Module Contents
===============
"""
-__version__ = "0.0.4"
+__version__ = "0.0.5"
__author__ = "Brennan Fieck"
import argparse
import datetime
-import sys
import logging
+import random
+import time
+
+from requests.exceptions import RequestException
+from trafficops.restapi import LoginError, OperationError, InvalidJSONError
def doMain(args:argparse.Namespace) -> int:
"""
@@ -177,49 +169,31 @@ def doMain(args:argparse.Namespace) -> int:
:returns: an exit code for the script.
:raises AttributeError: when the namespace is missing required arguments
"""
- from . import configuration
-
- if not configuration.setLogLevel(args.Log_Level):
- print("Unrecognized log level:", args.Log_Level, file=sys.stderr)
- return 1
-
- logging.info("Distribution detected as: '%s'", configuration.DISTRO)
- logging.info("Hostname detected as: '%s'", configuration.HOSTNAME[1])
-
- if not configuration.setMode(args.Mode):
- logging.critical("Unrecognized Mode: %s", args.Mode)
- return 1
-
- logging.info("Running in %s mode", configuration.MODE)
-
- if not configuration.setTSRoot(args.ts_root):
- logging.critical("Failed to set TS_ROOT, seemingly invalid path: '%s'", args.ts_root)
- return 1
-
- logging.info("ATS root installation directory set to: '%s'", configuration.TS_ROOT)
-
- configuration.VERIFY = not args.insecure
-
- if not configuration.setTOURL(args.Traffic_Ops_URL):
- logging.critical("Malformed or invalid Traffic_Ops_URL: '%s'", args.Traffic_Ops_URL)
+ from . import configuration, main_routines, to_api
+ random.seed(time.time())
+
+ try:
+ conf = configuration.Configuration(args)
+ except ValueError as e:
+ logging.critical(e)
+ logging.debug("%r", e, exc_info=True, stack_info=True)
return 1
- logging.info("Traffic Ops URL 'https://%s:%d' set and verified",
- configuration.TO_HOST, configuration.TO_PORT)
-
- if not configuration.setTOCredentials(args.Traffic_Ops_Login):
- logging.critical("Traffic Ops login credentials invalid or incorrect.")
+ if conf.login_dispersion:
+ disp = random.randint(0, conf.login_dispersion)
+ logging.info("Login dispersion is active - sleeping for %d seconds before continuing", disp)
+ time.sleep(disp)
+
+ try:
+ with to_api.API(conf) as api:
+ conf.api = api
+ return main_routines.run(conf)
+ except (LoginError, OperationError, InvalidJSONError, RequestException) as e:
+ logging.critical("Failed to connect and authenticate with the Traffic Ops server")
+ logging.error(e)
+ logging.debug("%r", e, exc_info=True, stack_info=True)
return 1
- #logging.info("Got TO Cookie - valid until %s",
- # datetime.datetime.fromtimestamp(configuration.TO_COOKIE.expires))
-
- configuration.WAIT_FOR_PARENTS = args.wait_for_parents
-
- from . import main_routines
-
- return main_routines.run()
-
def main():
"""
The ORT entrypoint, parses argv before handing it off to :func:`doMain`.
@@ -228,11 +202,13 @@ def main():
print(datetime.datetime.utcnow().strftime("%a %b %d %H:%M:%S UTC %Y"))
parser = argparse.ArgumentParser(description="A Python-based TO_ORT implementation",
+ epilog=("Note that passing a negative integer to options that "
+ "expect integers will instead set them to zero."),
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("Mode",
help="REPORT: Do nothing, but print what would be done\n"\
- "")
+ "REPORT, INTERACTIVE, REVALIDATE, SYNCDS, BADASS")
parser.add_argument("Log_Level",
help="ALL/TRACE, DEBUG, INFO, WARN, ERROR, FATAL/CRITICAL, NONE",
type=str)
@@ -255,8 +231,9 @@ def main():
default=3)
parser.add_argument("--wait_for_parents",
help="do not update if parent_pending = 1 in the update json.",
- action="store_true")
- parser.add_argument("--rev_proxy_disabled",
+ type=int,
+ default=1)
+ parser.add_argument("--rev_proxy_disable",
help="bypass the reverse proxy even if one has been configured.",
action="store_true")
parser.add_argument("--ts_root",
diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py
index 7e34ae3..9b3a89c 100644
--- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py
+++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/config_files.py
@@ -27,7 +27,11 @@ import typing
from base64 import b64decode
-from trafficops.restapi import OperationError, InvalidJSONError
+from trafficops.restapi import OperationError, InvalidJSONError, LoginError
+
+from .configuration import Configuration
+from .utils import getYesNoResponse as getYN
+
#: Holds a set of service names that need reloaded configs, mapped to a boolean which indicates
#: whether (:const:`True`) or not (:const:`False`) a full restart is required.
@@ -56,36 +60,37 @@ class ConfigFile():
contents = "" #: The full contents of the file - as configured in TO, not the on-disk contents
sanitizedContents = "" #: Will store the contents after sanitization
- def __init__(self, raw:dict = None):
+ def __init__(self, raw:dict = None, toURL:str = "", tsroot:str = "/"):
"""
Constructs a :class:`ConfigFile` object from a raw API response
:param raw: A raw config file from an API response
+ :param toURL: The URL of a valid Traffic Ops host
+ :param tsroot: The absolute path to the root of an Apache Traffic Server installation
:raises ValueError: if ``raw`` does not faithfully represent a configuration file
- >>> ConfigFile({"fnameOnDisk": "test",
- ... "location": "/path/to",
- ... "apiURI": "http://test",
- ... "scope": "servers"}))
- ConfigFile(path='/path/to/test', URI='http://test', scope='servers')
+ >>> a = ConfigFile({"fnameOnDisk": "test",
+ ... "location": "/path/to",
+ ... "apiURI":"/test",
+ ... "scope": servers}, "http://example.com/")
+ >>> a
+ ConfigFile(path='/path/to/test', URI='http://example.com/test', scope='servers')
+ >>> a.SSLdir
+ "/etc/trafficserver/ssl"
"""
- # TODO: pass these in as parameters? Configuration object?
- from .configuration import TO_HOST, TO_PORT, TO_USE_SSL, TS_ROOT
-
if raw is not None:
try:
self.fname = raw["fnameOnDisk"]
self.location = raw["location"]
if "apiUri" in raw:
- self.URI = "https://" if TO_USE_SSL else "http://"
- self.URI = "%s%s:%d/%s" % (self.URI, TO_HOST, TO_PORT, raw["apiUri"].lstrip('/'))
+ self.URI = toURL + raw["apiUri"].lstrip('/')
else:
self.URI = raw["url"]
self.scope = raw["scope"]
except (KeyError, TypeError, IndexError) as e:
raise ValueError from e
- self.SSLdir = os.path.join(TS_ROOT, "etc", "trafficserver", "ssl")
+ self.SSLdir = os.path.join(tsroot, "etc", "trafficserver", "ssl")
def __repr__(self) -> str:
"""
@@ -94,8 +99,8 @@ class ConfigFile():
>>> repr(ConfigFile({"fnameOnDisk": "test",
... "location": "/path/to",
... "apiURI": "http://test",
- ... "scope": "servers"}))
- "ConfigFile(path='/path/to/test', URI='http://test', scope='servers')"
+ ... "scope": "servers"}, "http://example.com/"))
+ "ConfigFile(path='/path/to/test', URI='http://example.com/test', scope='servers')"
"""
return "ConfigFile(path=%r, URI=%r, scope=%r)" %\
(self.path, self.URI if self.URI else None, self.scope)
@@ -127,53 +132,49 @@ class ConfigFile():
logging.info("fetched")
- def backup(self, contents:str):
+ def backup(self, contents:str, mode:Configuration.Modes):
"""
Creates a backup of this file under the :data:`BACKUP_DIR` directory
:param contents: The actual, on-disk contents from the original file
+ :param mode: The current run-mode of :program:`traffic_ops_ort`
:raises OSError: if the backup directory does not exist, or a backup of this file
could not be written into it.
"""
- from .configuration import MODE, Modes
- from .utils import getYesNoResponse
-
backupfile = os.path.join(BACKUP_DIR, self.fname)
willClobber = False
if os.path.isfile(backupfile):
willClobber = True
- if MODE is Modes.INTERACTIVE:
+ if mode is Configuration.Modes.INTERACTIVE:
prmpt = ("Write backup file %s%%s?" % backupfile)
prmpt %= " - will clobber existing file by the same name - " if willClobber else ''
- if not getYesNoResponse(prmpt, default='Y'):
+ if not getYN(prmpt, default='Y'):
return
elif willClobber:
logging.warning("Clobbering existing backup file '%s'!", backupfile)
- if MODE is not Modes.REPORT:
+ if mode is not Configuration.Modes.REPORT:
with open(backupfile, 'w') as fp:
fp.write(contents)
logging.info("Backup File written")
- def update(self, api:'to_api.API', cdn:str):
+ def update(self, conf:Configuration) -> bool:
"""
Updates the file if required, backing up as necessary
- :param api: A valid, authenticated API session for use when interacting with Traffic Ops
- :param cdn: The name of the CDN to which this server belongs (needed for SSL keys)
+ :param conf: An object that represents the configuration of :program:`traffic_ops_ort`
+ :returns: whether or not the file on disk actually changed
:raises OSError: when reading/writing files fails for some reason
"""
- from . import utils
- from .configuration import MODE, Modes, SERVER_INFO
from .services import NEEDED_RELOADS, FILES_THAT_REQUIRE_RELOADS
if not self.contents:
- self.fetchContents(api)
- finalContents = sanitizeContents(self.contents)
+ self.fetchContents(conf.api)
+ finalContents = sanitizeContents(self.contents, conf)
else:
finalContents = self.contents
@@ -184,83 +185,91 @@ class ConfigFile():
self.sanitizedContents = finalContents
if not os.path.isdir(self.location):
- if MODE is Modes.INTERACTIVE and\
- not utils.getYesNoResponse("Create configuration directory %s?" % self.path, 'Y'):
+ if (conf.mode is Configuration.Modes.INTERACTIVE and
+ not getYN("Create configuration directory %s?" % self.path, 'Y')):
logging.warning("%s will not be created - some services may not work properly!",
self.path)
- return
+ return False
logging.info("Directory %s will be created", self.location)
logging.info("File %s will be created", self.path)
- if MODE is not Modes.REPORT:
+ if conf.mode is not Configuration.Modes.REPORT:
os.makedirs(self.location)
with open(self.path, 'x') as fp:
fp.write(finalContents)
- return
+ return True
if not os.path.isfile(self.path):
- if MODE is Modes.INTERACTIVE and\
- not utils.getYesNoResponse("Create configuration file %s?"%self.path, default='Y'):
+ if (conf.mode is Configuration.Modes.INTERACTIVE and\
+ not getYN("Create configuration file %s?"%self.path, default='Y')):
logging.warning("%s will not be created - some services may not work properly!",
self.path)
- return
+ return False
logging.info("File %s will be created", self.path)
- if MODE is not Modes.REPORT:
+ if conf.mode is not Configuration.Modes.REPORT:
with open(self.path, 'x') as fp:
fp.write(finalContents)
- return
+ if self.fname == "ssl_multicert.config":
+ return self.advancedSSLProcessing(conf)
+ return True
+
+ written = False
with open(self.path, 'r+') as fp:
onDiskContents = fp.readlines()
if filesDiffer(finalContents.splitlines(), onDiskContents):
- self.backup(''.join(onDiskContents))
- if MODE is not Modes.REPORT:
+ self.backup(''.join(onDiskContents), conf.mode)
+ if conf.mode is not Configuration.Modes.REPORT:
fp.seek(0)
fp.truncate()
fp.write(finalContents)
- if self.fname in FILES_THAT_REQUIRE_RELOADS:
- NEEDED_RELOADS.add(FILES_THAT_REQUIRE_RELOADS[self.fname])
+
+ written = True
logging.info("File written to %s", self.path)
else:
logging.info("File doesn't differ from disk; nothing to do")
# Now we need to do some advanced processing to a couple specific filenames... unfortunately
if self.fname == "ssl_multicert.config":
- self.advancedSSLProcessing(api, cdn)
+ return self.advancedSSLProcessing(conf) or written
- def advancedSSLProcessing(self, api:'to_api.API', cdn:str):
+ return written
+
+ def advancedSSLProcessing(self, conf:Configuration):
"""
Does advanced processing on ssl_multicert.config files
- :param api: A valid, authenticated API session for use when interacting with Traffic Ops
- :param cdn: The name of the CDN to which this server belongs (needed for SSL keys)
+ :param conf: An object that represents the configuration of :program:`traffic_ops_ort`
:raises OSError: when reading/writing files fails for some reason
"""
global SSL_KEY_REGEX
- logging.info("Doing advanced SSL key processing for CDN '%s'", cdn)
+ logging.info("Doing advanced SSL key processing for CDN '%s'", conf.serverInfo.cdnName)
try:
- r = api.get_cdn_ssl_keys(cdn_name=cdn)
+ r = conf.api.get_cdn_ssl_keys(cdn_name=conf.serverInfo.cdnName)
if r[1].status_code != 200 and r[1].status_code != 204:
- raise ValueError("Bad response code: %d - raw response: %s" %
+ raise OSError("Bad response code: %d - raw response: %s" %
(r[1].status_code, r[1].text))
- except (OperationError, InvalidJSONError, ValueError) as e:
- logging.error("Invalid values encountered when communicating with Traffic Ops!")
- logging.debug("%r", e, stack_info=True, exc_info=True)
- raise ValueError from e
+ except (OperationError, LoginError, InvalidJSONError, ValueError) as e:
+ raise OSError("Invalid values encountered when communicating with Traffic Ops!") from e
logging.debug("Raw response from Traffic Ops: %s", r[1].text)
+ written = False
for l in self.sanitizedContents.splitlines()[1:]:
logging.debug("advanced processing for line: %s", l)
+
+ # for some reason, pylint is detecting this regular expression as a string
+ #pylint: disable=E1101
m = SSL_KEY_REGEX.search(l)
+ #pylint: enable=E1101
if m is None:
continue
@@ -281,28 +290,31 @@ class ConfigFile():
for cert in r[0]:
if cert.hostname == full or cert.hostname == wildcard:
- key = type(self)()
+ key = ConfigFile()
key.location = self.SSLdir
key.fname = m.group(2)
key.contents = b64decode(cert.certificate.key).decode()
logging.info("Processing private SSL key %s ...", key.fname)
- key.update(api, cdn)
+ written = key.update(conf)
logging.info("Done.")
- crt = type(self)()
+ crt = ConfigFile()
crt.location = self.SSLdir
crt.fname = m.group(1)
crt.contents = b64decode(cert.certificate.crt).decode()
logging.info("Processing SSL certificate %s ...", crt.fname)
- crt.update(api, cdn)
+ written = crt.update(conf)
logging.info("Done.")
break
else:
logging.critical("Failed to find SSL key in %s for '%s' or by wildcard '%s'!",
- cdn, full, wildcard)
- raise ValueError("No cert/key pair for ssl_multicert.config line '%s'" % l)
+ conf.serverInfo.cdnName, full, wildcard)
+ raise OSError("No cert/key pair for ssl_multicert.config line '%s'" % l)
+
+ # If even one key was written, we need to make ATS aware of the configuration changes
+ return written
def filesDiffer(a:typing.List[str], b:typing.List[str]) -> bool:
"""
@@ -328,22 +340,22 @@ def filesDiffer(a:typing.List[str], b:typing.List[str]) -> bool:
return False
-def sanitizeContents(raw:str) -> str:
+def sanitizeContents(raw:str, conf:Configuration) -> str:
"""
Performs pre-processing on a raw configuration file
:param raw: The raw contents of the file as returned by a request to its URL
+ :param conf: An object that represents the configuration of :program:`traffic_ops_ort`
:returns: The same contents, but with special replacement strings parsed out and HTML-encoded
symbols decoded to their literal values
"""
- from .configuration import SERVER_INFO
out = []
# These double curly braces escape the behaviour of Python's `str.format` method to attempt
# to use curly brace-enclosed text as a key into a dictonary of its arguments. They'll be
# rendered into single braces in the output of `.format`, leaving the string ultimately
# unchanged in that respect.
- for line in SERVER_INFO.sanitize(raw).splitlines():
+ for line in conf.serverInfo.sanitize(raw, conf.hostname).splitlines():
tmp=(" ".join(line.split())).strip() #squeezes spaces and trims leading and trailing spaces
tmp=tmp.replace("&", '&') #decodes HTML-encoded ampersands
tmp=tmp.replace(">", '>') #decodes HTML-encoded greater-than symbols
@@ -352,20 +364,20 @@ def sanitizeContents(raw:str) -> str:
return '\n'.join(out)
-def initBackupDir():
+def initBackupDir(mode:Configuration.Modes):
"""
Initializes a backup directory as a subdirectory of the directory containing
this ORT script.
+ :param mode: The current run-mode of :program:`traffic_ops_ort`
:raises OSError: if the backup directory initialization fails
"""
global BACKUP_DIR
- from . import configuration as conf
logging.info("Initializing backup dir %s", BACKUP_DIR)
if not os.path.isdir(BACKUP_DIR):
- if conf.MODE != conf.Modes.REPORT:
+ if mode is not Configuration.Modes.REPORT:
os.mkdir(BACKUP_DIR)
else:
logging.error("Cannot create non-existent backup dir in REPORT mode!")
diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/configuration.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/configuration.py
index 6f51990..a98d076 100644
--- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/configuration.py
+++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/configuration.py
@@ -22,71 +22,23 @@ hold and set up the log level, run modes, Traffic Ops login
credentials etc.
"""
+import argparse
import enum
import logging
import os
import platform
+import typing
import distro
import requests
-#: Contains the host's hostname as a tuple of ``(short_hostname, full_hostname)``
-HOSTNAME = (platform.node().split('.')[0], platform.node())
-#: contains identifying information about the host system's Linux distribution
-DISTRO = distro.LinuxDistribution().id()
-
-#: Holds information about the host system, required for processing configuration files,
-#: and also possibly useful in other situations
-SERVER_INFO = None
-
-#: This sets whether or not to verify SSL certificates when communicated with Traffic Ops.
-#: Does not affect non-Traffic Ops servers
-VERIFY = True
-
-#: If set to :const:`True`, this script will not apply updates until all of its parents have
-#: finished applying their updates
-WAIT_FOR_PARENTS = False
-
-
-class Modes(enum.IntEnum):
- """
- Enumerated run modes
- """
- REPORT = 0 #: Do nothing, only report what would be done
- INTERACTIVE = 1 #: Ask for user confirmation before modifying the system
- REVALIDATE = 2 #: Only check for configuration file changes and content revalidations
- SYNCDS = 3 #: Check for and apply Delivery Service changes
- BADASS = 4 #: Apply all settings specified in Traffic Ops, and attempt to solve all problems
-
- def __str__(self) -> str:
- """
- Implements ``str(self)`` by returning enum member's name
- """
- return self.name
-
-#: Holds the current run mode
-MODE = None
-
-def setMode(mode:str) -> bool:
- """
- Sets the script's run mode in the global variable :data:`MODE`
-
- :param mode: Expected to be the name of a :class:`Modes` constant.
- :returns: whether or not the run mode could be set successfully
- :raises ValueError: when ``mode`` is not a :const:`str`
- """
- try:
- mode = Modes[mode.upper()]
- except KeyError:
- return False
- except (AttributeError, ValueError) as e:
- raise ValueError from e
+#: A format specifier for logging output. Propagates to all imported modules.
+LOG_FORMAT = "%(levelname)s: %(asctime)s line %(lineno)d in %(module)s.%(funcName)s: %(message)s"
- global MODE
- MODE = mode
- return True
+#: contains identifying information about the host system's Linux distribution
+DISTRO = distro.LinuxDistribution().id()
class LogLevels(enum.IntEnum):
@@ -110,145 +62,187 @@ class LogLevels(enum.IntEnum):
"""
return self.name if self != logging.CRITICAL else "FATAL"
-#: A format specifier for logging output. Propagates to all imported modules.
-LOG_FORMAT = "%(levelname)s: %(asctime)s line %(lineno)d in %(module)s.%(funcName)s: %(message)s"
-def setLogLevel(level:str) -> bool:
+class Configuration():
"""
- Sets the global logger's log level to the desired name.
-
- :param level: Expected to be the name of a :class:`LogLevels` constant.
- :returns: whether or not the log level could be set successfully
- :raises ValueError: when the type of ``level`` is not :const:`str`
+ Represents a configured state for :program:`traffic_ops_ort`.
"""
- try:
- level = LogLevels[level.upper()]
- except KeyError:
- return False
- except (AttributeError, ValueError) as e:
- raise ValueError from e
- logging.basicConfig(level=level, format=LOG_FORMAT)
- logging.getLogger().setLevel(level)
+ class Modes(enum.IntEnum):
+ """
+ Enumerated representations for run modes for valid configurations.
+ """
+ REPORT = 0 #: Do nothing, only report what would be done
+ INTERACTIVE = 1 #: Ask for user confirmation before modifying the system
+ REVALIDATE = 2 #: Only check for configuration file changes and content revalidations
+ SYNCDS = 3 #: Check for and apply Delivery Service changes
+ BADASS = 4 #: Apply all settings specified in Traffic Ops, with no restrictions
- return True
+ def __str__(self) -> str:
+ """
+ Implements ``str(self)``
+ :returns: the enum member's name
+ """
+ return self.name
-#: An absolute path to the root installation directory of the Apache Trafficserver installation
-TS_ROOT = None
-def setTSRoot(tsroot:str) -> bool:
- """
- Sets the global variable :data:`TS_ROOT`.
+ #: Holds a reference to a :class:`to_api.API` object used by this configuration - must be set
+ #: manually.
+ api = None
+
+
+ #: Holds a reference to a :class:`to_api.ServerInfo` object used by this configuration - must be
+ #: set manually.
+ ServerInfo = None
+
+
+ def __init__(self, args:argparse.Namespace):
+ """
+ Constructs the configuration object.
+
+ :param args: Should be the result of parsing command-line arguments to :program:`traffic_ops_ort`
+ :raises ValueError: if an error occurred setting up the configuration
+ """
+ global DISTRO
+
+ self.dispersion = args.dispersion if args.dispersion > 0 else 0
+ self.login_dispersion = args.login_dispersion if args.login_dispersion > 0 else 0
+ self.wait_for_parents = bool(args.wait_for_parents)
+ self.retries = args.retries if args.retries > 0 else 0
+ self.rev_proxy_disable = args.rev_proxy_disable
+ self.verify = not args.insecure
+
+ setLogLevel(args.Log_Level)
+
+ logging.info("Distribution detected as: '%s'", DISTRO)
- :param tsroot: Should be an absolute path to the directory containing the system's Apache
- Trafficserver installation.
- :returns: whether or not the installation path could be set successfully
- :raises ValueError: if ``tsroot`` is not a :const:`str`
+ self.hostname = (platform.node().split('.')[0], platform.node())
+ logging.info("Hostname detected as: '%s'", self.fullHostname)
+ try:
+ self.mode = Configuration.Modes[args.Mode.upper()]
+ except KeyError as e:
+ raise ValueError("Unrecognized Mode: '%s'" % args.Mode)
+
+ self.tsroot = parseTSRoot(args.ts_root)
+ logging.info("ATS root installation directory set to '%s'", self.tsroot)
+
+ self.useSSL, self.toHost, self.toPort = parseTOURL(args.Traffic_Ops_URL, self.verify)
+
+ try:
+ self.username, self.password = args.Traffic_Ops_Login.split(':')
+ except ValueError as e:
+ raise ValueError("Invalid login information, should be like 'username:password'.") from e
+
+
+ @property
+ def shortHostname(self) -> str:
+ """
+ Convenience accessor for the short hostname of this server
+
+ :returns: The (short) hostname of this server as detected by :func:`platform.node`
+ """
+ return self.hostname[0]
+
+ @property
+ def fullHostname(self) -> str:
+ """
+ Convenience accessor for the full hostname of this server
+
+ :returns: The hostname of this server as detected by :func:`platform.node`
+ """
+ return self.hostname[1]
+
+ @property
+ def TOURL(self) -> str:
+ """
+ Convenience function to construct a full URL out of whatever information was given at runtime
+
+ :returns: The configuration's URL which points to its Traffic Ops server instance
+
+ .. note:: This is totally constructed from information given on the command line; the
+ resulting URL may actually point to a reverse proxy for the Traffic Ops server and not
+ the server itself.
+ """
+ return "%s://%s:%d/" % ("https" if self.useSSL else "https", self.toHost, self.toPort)
+
+
+def setLogLevel(level:str):
"""
+ Parses a string to return the requested :class:`LogLevels` member, to which it will then set
+ the global logging level.
+
+ :param level: the name of a LogLevels enum constant
+ :raises ValueError: if ``level`` cannot be parsed to an actual LogLevel
+ """
+ global LOG_FORMAT
+
try:
- tsroot = tsroot.strip()
+ level = LogLevels[level.upper()]
+ except KeyError as e:
+ raise ValueError("Unrecognized log level: '%s'" % level) from e
- if tsroot != '/' and tsroot.endswith('/'):
- tsroot = tsroot.rstrip('/')
+ logging.basicConfig(level=level, format=LOG_FORMAT)
+ logging.getLogger().setLevel(level)
- if not os.path.isdir(tsroot) or\
- not os.path.isfile(os.path.join(tsroot, 'bin', 'trafficserver')):
- return False
- except (OSError, AttributeError, ValueError) as e:
- raise ValueError from e
+def parseTSRoot(tsroot:str) -> str:
+ """
+ Parses and validates a given path as a path to the root of an Apache Traffic Server installation
- global TS_ROOT
- TS_ROOT = tsroot
- return True
+ :param tsroot: The relative or absolute path to the root of this server's ATS installation
+ :raises ValueError: if ``tsroot`` is not an existing path, or does not contain the ATS binary
+ """
+ tsroot = tsroot.strip()
+ if tsroot != '/' and tsroot.endswith('/'):
+ tsroot = tsroot.rstrip('/')
+ try:
+ if not os.path.isdir(tsroot):
+ raise ValueError("'%s' is not a directory!" % tsroot)
-#: :const:`True` if Traffic Ops communicates using SSL, :const:`False` otherwise
-TO_USE_SSL = False
+ binpath = os.path.join(tsroot, 'bin', 'trafficserver')
+ if not os.path.isfile(binpath):
+ raise ValueError("'%s' does not exist! '%s' is not the root of a Traffic Server"
+ "installation" % (binpath, tsroot))
+ except OSError as e:
+ raise ValueError("Couldn't set the ATS root install directory: %s" % e) from e
-#: Holds only the :abbr:`FQDN (Fully Quallified Domain Name)` of the Traffic Ops server
-TO_HOST = None
+ return tsroot
-#: Holds the port number on which the Traffic Ops server listens for incoming HTTP(S) requests
-TO_PORT = None
-def setTOURL(url:str) -> bool:
+def parseTOURL(url:str, verify:bool) -> typing.Tuple[bool, str, int]:
"""
- Sets the :data:`TO_USE_SSL`, :data:`TO_PORT` and :data:`TO_HOST` global variables and verifies,
- them.
+ Parses and verifies the passed URL and breaks it into parts for the caller
- :param url: A full URL (including schema - and port when necessary) specifying the location of
- a running Traffic Ops server
- :returns: whether or not the URL could be set successfully
- :raises ValueError: when ``url`` is not a :const:`str`
+ :param url: At minimum an FQDN for a Traffic Ops server, but can include schema and port number
+ :param verify: Whether or not to verify the server's SSL certificate
+ :returns: Whether or not the Traffic Ops server uses SSL (http vs https), the server's FQDN, and the port on which it listens
+ :raises ValueError: if ``url`` does not point at a valid HTTP server or is incorrectly formatted
"""
- global VERIFY
- try:
- url = url.rstrip('/')
- _ = requests.head(url, verify=VERIFY)
- except requests.exceptions.RequestException as e:
- logging.error("%s", e)
- logging.debug("%s", e, exc_info=True, stack_info=True)
- return False
- except (AttributeError, ValueError) as e:
- raise ValueError from e
+ url = url.rstrip('/')
- global TO_HOST, TO_PORT, TO_USE_SSL
+ useSSL, host, port = True, None, 443
- port = None
+ try:
+ _ = requests.head(url, verify=verify)
+ except requests.exceptions.RequestException as e:
+ raise ValueError("Cannot contact any server at '%s' (%s)" % (url, e)) from e
if url.lower().startswith("http://"):
- url = url[7:]
port = 80
- TO_USE_SSL = False
+ useSSL = False
+ url = url[7:]
elif url.lower().startswith("https://"):
url = url[8:]
- port = 443
- TO_USE_SSL = True
- # I'm assuming here that a valid FQDN won't include ':' - and it shouldn't.
+ # I'm assuming here that a valid FQDN won't include ':' - and it shouldn't
portpoint = url.find(':')
if portpoint > 0:
- TO_HOST = url[:portpoint]
+ host = url[:portpoint]
port = int(url[portpoint+1:])
else:
- TO_HOST = url
-
- if port is None:
- raise ValueError("Couldn't determine port number from URL '%s'!" % url)
-
- TO_PORT = port
-
- return True
-
-#: Holds the username used for logging in to Traffic Ops
-USERNAME = None
+ host = url
-#: Holds the password used to authenticate :data:`USERNAME` with Traffic Ops
-PASSWORD = None
-
-def setTOCredentials(login:str) -> bool:
- """
- Parses and returns a JSON-encoded login string for the Traffic Ops API.
- This will set :data:`USERNAME` and :data:`PASSWORD` if login is successful.
-
- :param login: The raw login info as passed on the command line (e.g. 'username:password')
- :raises ValueError: if ``login`` is not a :const:`str`.
- :returns: whether or not the login could be set and validated successfully
- """
- try:
- u, p = login.split(':')
- login = '{{"u": "{0}", "p": "{1}"}}'.format(u, p)
- except IndexError:
- logging.critical("Bad Traffic_Ops_Login: '%s' - should be like 'username:password'", login)
- return False
- except (AttributeError, ValueError) as e:
- raise ValueError from e
-
- global USERNAME, PASSWORD
- USERNAME = u
- PASSWORD = p
-
- return True
+ return useSSL, host, port
diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py
index 91eab2a..b94794c 100644
--- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py
+++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/main_routines.py
@@ -22,11 +22,11 @@ and performs a variety of operations based on the run mode.
import os
import logging
-import requests
+import random
+import time
-from trafficops.restapi import LoginError, OperationError
-
-from . import to_api
+from .configuration import Configuration
+from .utils import getYesNoResponse as getYN
#: A constant that holds the absolute path to the status file directory
STATUS_FILE_DIR = "/opt/ort/status"
@@ -35,86 +35,80 @@ class ORTException(Exception):
"""Signifies an ORT related error"""
pass
-def syncDSState(api:to_api.API) -> bool:
+def syncDSState(conf:Configuration) -> bool:
"""
Queries Traffic Ops for the :term:`Delivery Service`'s sync state
- :param api: A :class:`traffic_ops_ort.to_api.API` object to use when interacting with Traffic Ops
+ :param conf: The script's configuration
- :raises ORTException: when something goes wrong
:returns: whether or not an update is needed
+ :raises ConnectionError: when something goes wrong communicating with Traffic Ops
"""
- from . import configuration
logging.info("starting syncDS state fetch")
- try:
- updateStatus = api.getUpdateStatus(api.hostname)[0]
- except (IndexError, ConnectionError, requests.exceptions.RequestException) as e:
- logging.critical("Server configuration not found in Traffic Ops!")
- raise ORTException from e
- except PermissionError as e:
- logging.critical("Failed to authenticate with the Traffic Ops server!")
- raise ORTException from e
+ updateStatus = conf.api.getMyUpdateStatus()[0]
logging.debug("Retrieved raw update status: %r", updateStatus)
- return 'upd_pending' in updateStatus and updateStatus['upd_pending']
+ if 'upd_pending' not in updateStatus:
+ raise ConnectionError("Malformed API response doesn't indicate if updates are pending!")
+
+ if not updateStatus['upd_pending']:
+ return False
+
+ if conf.wait_for_parents and 'parent_pending' in updateStatus and updateStatus["parent_pending"]:
+ logging.warning("One or more parents still have updates pending, waiting for parents.")
+ return False
+
+ if conf.mode is Configuration.Modes.SYNCDS and conf.dispersion:
+ disp = random.randint(0, conf.dispersion)
+ logging.info("Dispersion is set. Will sleep for %d seconds before continuing", disp)
+ time.sleep(disp)
+
+ return True
-def revalidateState(api:to_api.API) -> bool:
+def revalidateState(conf:Configuration) -> bool:
"""
Checks the revalidation status of this server in Traffic Ops
- :param api: A :class:`traffic_ops_ort.to_api.API` object to use when interacting with Traffic Ops
+ :param conf: The script's configuration
:returns: whether or not this server has a revalidation pending
- :raises ORTException:
+ :raises ConnectionError: when something goes wrong communicating with Traffic Ops
"""
- from . import configuration as conf
logging.info("starting revalidation state fetch")
- try:
- updateStatus = api.getUpdateStatus(api.hostname)
- except (IndexError, ConnectionError, requests.exceptions.RequestException) as e:
- logging.critical("Server configuration not found in Traffic Ops!")
- raise ORTException from e
- except PermissionError as e:
- logging.critical("Failed to authenticate with the Traffic Ops server!")
- raise ORTException from e
+ updateStatus = conf.api.getMyUpdateStatus()[0]
logging.debug("Retrieved raw revalidation status: %r", updateStatus)
- if conf.WAIT_FOR_PARENTS and\
- "parent_reval_pending" in updateStatus and\
- updateStatus["parent_reval_pending"]:
+ if (conf.wait_for_parents and
+ "parent_reval_pending" in updateStatus and
+ updateStatus["parent_reval_pending"]):
logging.info("Parent revalidation is pending - waiting for parent")
return False
return "reval_pending" in updateStatus and updateStatus["reval_pending"]
-def deleteOldStatusFiles(myStatus:str, api:to_api.API):
+def deleteOldStatusFiles(myStatus:str, conf:Configuration):
"""
Attempts to delete any and all old status files
:param myStatus: the current status - files by this name will not be deleted
- :param api: A :class:`traffic_ops_ort.to_api.API` object to use when interacting with Traffic Ops
- :raises ConnectionError: if there's an issue retrieving a list of statuses from
- Traffic Ops
+ :param conf: An object containing the configuration of :program:`traffic_ops_ort`
+ :raises ConnectionError: if there's an issue retrieving a list of statuses from Traffic Ops
:raises OSError: if a file cannot be deleted for any reason
"""
- from .configuration import MODE, Modes
- from . import utils
-
logging.info("Deleting old status files (those that are not %s)", myStatus)
- doDeleteFiles = MODE is not Modes.REPORT
+ doDeleteFiles = conf.mode is not Configuration.Modes.REPORT
- for status in api.get_statuses()[0]:
+ for status in conf.api.get_statuses()[0]:
# Only the status name matters
try:
status = status.name
except KeyError as e:
logging.debug("Bad status object: %r", status)
- logging.debug("Original error: %s", e, exc_info=True, stack_info=True)
raise ConnectionError from e
if doDeleteFiles and status != myStatus:
@@ -124,52 +118,45 @@ def deleteOldStatusFiles(myStatus:str, api:to_api.API):
logging.info("File '%s' to be deleted", fname)
# check for user confirmation before deleting files in 'INTERACTIVE' mode
- if MODE != Modes.INTERACTIVE or utils.getYesNoResponse("Delete file %s?" % fname):
+ if conf.mode is not Configuration.Modes.INTERACTIVE or getYN("Delete file %s?" % fname):
logging.warning("Deleting file '%s'!", fname)
os.remove(fname)
-def setStatusFile(api:to_api.API) -> bool:
+def setStatusFile(conf:Configuration) -> bool:
"""
Attempts to set the status file according to this server's reported status in Traffic Ops.
.. warning:: This will create the directory '/opt/ORTstatus' if it does not exist, and may
delete files there without warning!
- :param api: A :class:`traffic_ops_ort.to_api.API` object to use when interacting with Traffic Ops
+ :param conf: An object that contains the configuration for :program:`traffic_ops_ort`
:returns: whether or not the status file could be set properly
"""
global STATUS_FILE_DIR
- from .configuration import MODE, Modes
- from . import utils
logging.info("Setting status file")
- if not isinstance(MODE, Modes):
- logging.error("MODE is not set to a valid Mode (from traffic_ops_ort.configuration.Modes)!")
- return False
-
try:
- myStatus = api.getMyStatus()
+ myStatus = conf.api.getMyStatus()
except ConnectionError as e:
logging.error("Failed to set status file - Traffic Ops connection failed")
return False
if not os.path.isdir(STATUS_FILE_DIR):
logging.warning("status directory does not exist, creating...")
- doMakeDir = MODE is not Modes.REPORT
+ doMakeDir = conf.mode is not Configuration.Modes.REPORT
# Check for user confirmation if in 'INTERACTIVE' mode
- if doMakeDir and (MODE is not Modes.INTERACTIVE or\
- utils.getYesNoResponse("Create status directory '%s'?" % STATUS_FILE_DIR, default='Y')):
+ if doMakeDir and (conf.mode is not Configuration.Modes.INTERACTIVE or
+ getYN("Create status directory '%s'?" % STATUS_FILE_DIR, default='Y')):
try:
os.makedirs(STATUS_FILE_DIR)
- return False
except OSError as e:
logging.error("Failed to create status directory '%s' - %s", STATUS_FILE_DIR, e)
logging.debug("%s", e, exc_info=True, stack_info=True)
return False
else:
try:
- deleteOldStatusFiles(myStatus, api)
+ deleteOldStatusFiles(myStatus, conf)
except ConnectionError as e:
logging.error("Failed to delete old status files - Traffic Ops connection failed.")
logging.debug("%s", e, exc_info=True, stack_info=True)
@@ -182,8 +169,8 @@ def setStatusFile(api:to_api.API) -> bool:
fname = os.path.join(STATUS_FILE_DIR, myStatus)
if not os.path.isfile(fname):
logging.info("File '%s' to be created", fname)
- if MODE is not Modes.REPORT and\
- (MODE is not Modes.INTERACTIVE or utils.getYesNoResponse("Create file '%s'?", 'y')):
+ if conf.mode is not Configuration.Modes.REPORT and (
+ conf.mode is not Configuration.Modes.INTERACTIVE or getYN("Create file '%s'?", 'y')):
try:
with open(fname, 'x'):
@@ -195,62 +182,68 @@ def setStatusFile(api:to_api.API) -> bool:
return True
-def processPackages(api:to_api.API) -> bool:
+def processPackages(conf:Configuration) -> bool:
"""
Manages the packages that Traffic Ops reports are required for this server.
- :param api: A :class:`traffic_ops_ort.to_api.API` object to use when interacting with Traffic Ops
+ :param conf: An object containing the configuration of :program:`traffic_ops_ort`
:returns: whether or not the package processing was successfully completed
"""
- from .configuration import Modes, MODE
-
try:
- myPackages = api.getMyPackages()
- except (ConnectionError, PermissionError) as e:
- logging.error("Failed to fetch package list from Traffic Ops - %s", e)
- logging.debug("%s", e, exc_info=True, stack_info=True)
- return False
- except ValueError as e:
- logging.error("Got malformed response from Traffic Ops! - %s", e)
+ myPackages = conf.api.getMyPackages()
+ except ConnectionError as e:
+ logging.error("Packages not found or API response malformed! - %s", e)
logging.debug("%s", e, exc_info=True, stack_info=True)
return False
for package in myPackages:
- if package.install():
- if MODE is not Modes.BADASS:
+ if package.install(conf):
+ if conf.mode is not Configuration.Modes.BADASS:
return False
logging.warning("Failed to install %s, but we're BADASS, so moving on!", package)
return True
-def processServices(api:to_api.API) -> bool:
+def processServices(conf:Configuration) -> bool:
"""
Manages the running processes of the server, according to an ancient system known as 'chkconfig'
- :param api: A :class:`traffic_ops_ort.to_api.API` object to use when interacting with Traffic Ops
+ :param conf: An object containing the configuration for :program:`traffic_ops_ort`
:returns: whether or not the service processing was completed successfully
"""
from . import services
- for item in api.getMyChkconfig():
+ if not services.HAS_SYSTEMD:
+ logging.warning("This system doesn't have systemd, services cannot be enabled/disabled")
+ return True
+
+
+ try:
+ chkconfig = conf.api.getMyChkconfig()
+ except ConnectionError as e:
+ logging.error("Failed to fetch 'chkconfig' from Traffic Ops! (%s)", e)
+ logging.debug("%r", e, exc_info=True, stack_info=True)
+ return False
+
+ for item in chkconfig:
logging.debug("Processing item %r", item)
- if not services.setServiceStatus(item):
+ if not services.setServiceStatus(item, conf.mode):
return False
return True
-def processConfigurationFiles(api:to_api.API) -> bool:
+def processConfigurationFiles(conf:Configuration) -> bool:
"""
Updates and backs up all of a server's configuration files.
- :param api: A :class:`traffic_ops_ort.to_api.API` object to use when interacting with Traffic Ops
+ :param conf: An object containing the configuration for :program:`traffic_ops_ort`
:returns: whether or not the configuration changes were successful
"""
- from . import config_files, configuration
+ from . import config_files, services
try:
- config_files.initBackupDir()
+ config_files.initBackupDir(conf.mode)
except OSError as e:
logging.error("Couldn't create backup directory!")
logging.warning("%s", e)
@@ -258,28 +251,25 @@ def processConfigurationFiles(api:to_api.API) -> bool:
return False
try:
- myFiles = api.getMyConfigFiles()
+ myFiles = conf.api.getMyConfigFiles(conf)
except ConnectionError as e:
- logging.error("Failed to fetch configuration files - Traffic Ops connection failed! %s",e)
- logging.debug("%s", e, exc_info=True, stack_info=True)
- return False
- except ValueError as e:
- logging.error("Malformed configuration file response from Traffic Ops!")
+ logging.critical("Failed to fetch configuration files; Traffic Ops connection failed! %s",e)
logging.debug("%s", e, exc_info=True, stack_info=True)
return False
for file in myFiles:
try:
- file = config_files.ConfigFile(file)
+ file = config_files.ConfigFile(file, conf.TOURL)
logging.info("\n============ Processing File: %s ============", file.fname)
- file.update(api, configuration.SERVER_INFO.cdnName)
+ if file.update(conf) and file.fname in services.FILES_THAT_REQUIRE_RELOADS:
+ services.NEEDED_RELOADS.add(services.FILES_THAT_REQUIRE_RELOADS[file.fname])
logging.info("\n============================================\n")
# A bad object could just reflect an inconsistent reply structure from the API, so BADASSes
# will attempt to continue. However, an issue updating a valid configuration is not
# recoverable, even for BADASSes
- except config_files.ConfigurationError as e:
- logging.error("An error occurred while trying to update %s", file.name)
+ except OSError as e:
+ logging.error("An error occurred while trying to update %s", file.fname)
logging.debug("%s", e, exc_info=True, stack_info=True)
return False
except ValueError as e:
@@ -289,30 +279,24 @@ def processConfigurationFiles(api:to_api.API) -> bool:
return True
-def run() -> int:
+def run(conf:Configuration) -> int:
"""
This function is the entrypoint into the script's main flow from :func:`traffic_ops_ort.doMain`
- It runs the appropriate actions depending on the run mode
+ It runs the appropriate actions depending on the run mode.
+
+ :param conf: An object that holds the script's configuration
:returns: an exit code for the script
"""
- from . import configuration, utils, services
-
- try:
- api = to_api.API(configuration.USERNAME, configuration.PASSWORD, configuration.TO_HOST,
- configuration.HOSTNAME[0], configuration.TO_PORT, configuration.VERIFY,
- configuration.TO_USE_SSL)
- except (LoginError, OperationError) as e:
- logging.critical("Failed to authenticate with Traffic Ops")
- logging.error(e)
- logging.debug("%r", e, exc_info=True, stack_info=True)
- return 1
+ from . import services
# If this is just a revalidation, then we can exit if there's no revalidation pending
- if configuration.MODE == configuration.Modes.REVALIDATE:
+ if conf.mode is Configuration.Modes.REVALIDATE:
try:
- updateRequired = revalidateState(api)
- except ORTException as e:
+ updateRequired = revalidateState(conf)
+ except ConnectionError as e:
+ logging.critical("Server configuration unreachable, or not found in Traffic Ops!")
+ logging.error(e)
logging.debug("%r", e, exc_info=True, stack_info=True)
return 2
@@ -326,30 +310,32 @@ def run() -> int:
# changes
else:
try:
- updateRequired = syncDSState(api)
- except ORTException as e:
+ updateRequired = syncDSState(conf)
+ except ConnectionError as e:
+ logging.critical("Server configuration unreachable, or not found in Traffic Ops!")
+ logging.error(e)
logging.debug("%r", e, exc_info=True, stack_info=True)
return 2
# Bail on failures - unless this script is BADASS!
- if not setStatusFile(api):
- if configuration.MODE is not configuration.Modes.BADASS:
+ if not setStatusFile(conf):
+ if conf.mode is not Configuration.Modes.BADASS:
logging.critical("Failed to set status as specified by Traffic Ops")
return 2
logging.warning("Failed to set status but we're BADASS, so moving on.")
logging.info("\nProcessing Packages...")
- if not processPackages(api):
+ if not processPackages(conf):
logging.critical("Failed to process packages")
- if configuration.MODE is not configuration.Modes.BADASS:
+ if conf.mode is not Configuration.Modes.BADASS:
return 2
logging.warning("Package processing failed but we're BADASS, so attempting to move on")
logging.info("Done.\n")
logging.info("\nProcessing Services...")
- if not processServices(api):
+ if not processServices(conf):
logging.critical("Failed to process services.")
- if configuration.MODE is not configuration.Modes.BADASS:
+ if conf.mode is not Configuration.Modes.BADASS:
return 2
logging.warning("Service processing failed but we're BADASS, so attempting to move on")
logging.info("Done.\n")
@@ -357,17 +343,17 @@ def run() -> int:
# All modes process configuration files
logging.info("\nProcessing Configuration Files...")
- if not processConfigurationFiles(api):
+ if not processConfigurationFiles(conf):
logging.critical("Failed to process configuration files.")
return 2
logging.info("Done.\n")
if updateRequired:
- if (configuration.MODE is not configuration.Modes.INTERACTIVE or
- utils.getYesNoResponse("Update Traffic Ops?", default='Y')):
+ if (conf.mode is not Configuration.Modes.INTERACTIVE or
+ getYN("Update Traffic Ops?", default='Y')):
logging.info("\nUpdating Traffic Ops...")
- api.updateTrafficOps()
+ conf.api.updateTrafficOps(conf.mode)
logging.info("Done.\n")
else:
logging.warning("Traffic Ops was not notified of changes. You should do this manually.")
@@ -375,7 +361,7 @@ def run() -> int:
else:
logging.info("Traffic Ops update not necessary")
- if services.NEEDED_RELOADS and not services.doReloads():
+ if services.NEEDED_RELOADS and not services.doReloads(conf):
logging.critical("Failed to reload all configuration changes")
return 2
diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/packaging.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/packaging.py
index d19fa24..7d15dbc 100644
--- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/packaging.py
+++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/packaging.py
@@ -24,6 +24,9 @@ support a strict set of distributions with well-known package managers.
import logging
import subprocess
+from .configuration import Configuration
+from .utils import getYesNoResponse as getYN
+
class _MetaPackage(type):
"""
This factory is responsible for constructing a :class:`Package` class which properly
@@ -126,26 +129,25 @@ class Package(metaclass=_MetaPackage):
return True
return False
- def install(self) -> int:
+ def install(self, conf:Configuration) -> int:
"""
Installs this package.
+ :param conf: An object containing the configuration for :program:`traffic_ops_ort`
+
:returns: the exit code of the install process
"""
- from .configuration import MODE, Modes
- from .utils import getYesNoResponse as getYN
-
if self.isInstalled():
logging.info("%s is already installed - nothing to do", self)
return 0
- if MODE is Modes.INTERACTIVE and not getYN("Install %s?" % self, default='Y'):
+ if conf.mode is Configuration.Modes.INTERACTIVE and not getYN("Install %s?" % self, 'Y'):
logging.warning("%s will not be installed, dependencies may be unsatisfied!", self)
return 0
logging.info("Installing %s", self)
- if MODE is Modes.REPORT:
+ if conf.mode is Configuration.Modes.REPORT:
return 0
try:
@@ -167,26 +169,23 @@ class Package(metaclass=_MetaPackage):
return sub.returncode
- def uninstall(self) -> int:
+ def uninstall(self, conf:Configuration) -> int:
"""
- Uninstalls this package
+ Uninstalls this package. I have no idea how one would make use of this from within ATC...
:returns: the exit code of the uninstall process
"""
- from .configuration import MODE, Modes
- from .utils import getYesNoResponse as getYN
-
if not self.isInstalled():
logging.info("%s is not installed - nothing to do", self)
return 0
- if MODE is Modes.INTERACTIVE and not getYN("Uninstall %s?" % self, default='Y'):
+ if conf.mode is Configuration.Modes.INTERACTIVE and not getYN("Uninstall %s?" % self, 'Y'):
logging.warning("%s will not be installed, dependencies may be out of date!", self)
return 0
logging.info("Uninstalling %s", self)
- if MODE is Modes.REPORT:
+ if conf.mode is Configuration.Modes.REPORT:
return 0
try:
diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/services.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/services.py
index a725d96..402806c 100644
--- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/services.py
+++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/services.py
@@ -28,8 +28,14 @@ import logging
import os
import subprocess
import typing
+
+from functools import partial
+
import psutil
+from .configuration import Configuration
+from .utils import getYesNoResponse as getYN
+
#: Holds the list of reloads needed due to configuration file changes
NEEDED_RELOADS = set()
@@ -43,34 +49,33 @@ except subprocess.CalledProcessError:
else:
HAS_SYSTEMD = True
-def reloadATSConfigs() -> bool:
+def reloadATSConfigs(conf:Configuration) -> bool:
"""
This function will reload configuration files for the Apache Trafficserver caching HTTP
proxy. It does this by calling ``traffic_ctl config reload`
+ :param conf: An object representing the configuration of :program:`traffic_ops_ort`
:returns: whether or not the reload succeeded (as indicated by the exit code of
``traffic_ctl``)
:raises OSError: when something goes wrong executing the child process
"""
- from .configuration import MODE, Modes, TS_ROOT
- from .utils import getYesNoResponse as getYN
-
# First of all, ATS must be running for this to work
- if not setATSStatus(True):
+ if not setATSStatus(True, conf):
logging.error("Cannot reload configs, ATS not running!")
return False
- cmd = [os.path.join(TS_ROOT, "bin", "traffic_ctl"), "config", "reload"]
+ cmd = [os.path.join(conf.tsroot, "bin", "traffic_ctl"), "config", "reload"]
cmdStr = ' '.join(cmd)
- if MODE is Modes.INTERACTIVE and not getYN("Run command '%s' to reload configuration?",cmdStr):
+ if ( conf.mode is Configuration.Modes.INTERACTIVE and
+ not getYN("Run command '%s' to reload configuration?" % cmdStr, default='Y')):
logging.warning("Configuration will not be reloaded for Apache Trafficserver!")
logging.warning("Changes will NOT be applied!")
return True
logging.info("Apache Trafficserver configuration reload will be done via: %s", cmdStr)
- if MODE is Modes.REPORT:
+ if conf.mode is Configuration.Modes.REPORT:
return True
sub = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -85,18 +90,50 @@ def reloadATSConfigs() -> bool:
return False
return True
-def restartATS() -> bool:
+def restartATS(conf:Configuration) -> bool:
"""
A convenience function for calling :func:`setATSStatus` for restarts.
+ :param conf: An object representing the configuration of :program:`traffic_ops_ort`
:returns: whether or not the restart was successful (or unnecessary)
"""
- from .configuration import MODE, Modes
- from .utils import getYesNoResponse as getYN
- doRestart = MODE is Modes.BADASS or MODE is Modes.REPORT or (MODE is Modes.INTERACTIVE and
- getYN("Restart ATS?", default='Y'))
- return setATSStatus(True, restart=doRestart)
+ doRestart = ( conf.mode is Configuration.Modes.BADASS or
+ conf.mode is Configuration.Modes.REPORT or
+ ( conf.mode is Configuration.Modes.INTERACTIVE and
+ getYN("Restart ATS?", default='Y')))
+
+ return setATSStatus(True, conf, restart=doRestart)
+
+
+def restartService(service:str, conf:Configuration) -> bool:
+ """
+ Restarts a generic systemd service
+
+ :param service: The name of the service to be restarted
+ :param conf: An object representing the configuration of :program:`traffic_ops_ort`
+ :returns: Whether or not the restart was successful
+ """
+ global HAS_SYSTEMD
+
+ if not HAS_SYSTEMD:
+ logging.warning("This system doesn't have systemd, services cannot be restarted")
+ return True
+
+ if conf.mode is not Configuration.Modes.REPORT and (
+ conf.mode is not Configuration.Modes.INTERACTIVE or getYN("Restart %s?" % service, 'Y')):
+ logging.info("Restarting %s", service)
+ try:
+ sub = subprocess.Popen(["systemctl", "restart", service],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = sub.communicate()
+ logging.debug("stdout: %s\nstderr: %s", out, err)
+ except (OSError, subprocess.CalledProcessError) as e:
+ logging.error("An error occurred when restarting %s: %s", service, e)
+ logging.debug("%r", e, exc_info=True, stack_info=True)
+ return False
+ return True
#: A big ol' map of filenames to the services which require reloads when said files change
FILES_THAT_REQUIRE_RELOADS = {"records.config": reloadATSConfigs,
@@ -108,13 +145,14 @@ FILES_THAT_REQUIRE_RELOADS = {"records.config": reloadATSConfigs,
"logs_xml.config": reloadATSConfigs,
"ssl_multicert.config": reloadATSConfigs,
"plugin.config": restartATS,
- "ntpd.conf": lambda: restartService("ntpd"),
+ "ntpd.conf": partial(restartService, "ntpd"),
"50-ats.rules": restartATS}
-def doReloads() -> bool:
+def doReloads(conf:Configuration) -> bool:
"""
Performs all necessary service restarts/configuration reloads
+ :param conf: An object representing the configuration of :program:`traffic_ops_ort`
:returns: whether or not the reloads/restarts went successfully
"""
global NEEDED_RELOADS
@@ -125,7 +163,7 @@ def doReloads() -> bool:
for reload in NEEDED_RELOADS:
try:
- if not reload():
+ if not reload(conf):
return False
except OSError as e:
logging.error("An error occurred when reloading service configuration files: %s",e)
@@ -158,7 +196,7 @@ def getProcessesIfRunning(name:str) -> typing.Optional[psutil.Process]:
return None
-def setATSStatus(status:bool, restart:bool = False) -> bool:
+def setATSStatus(status:bool, conf:Configuration, restart:bool = False) -> bool:
"""
Sets the status of the system's ATS process.
@@ -170,9 +208,6 @@ def setATSStatus(status:bool, restart:bool = False) -> bool:
:returns: whether or not the status setting was successful (or unnecessary)
:raises OSError: when there is a problem executing the subprocess
"""
- from .configuration import MODE, Modes, TS_ROOT
- from .utils import getYesNoResponse as getYN
-
existingProcess = getProcessesIfRunning("[TS_MAIN]")
# ATS is not running
@@ -205,14 +240,15 @@ def setATSStatus(status:bool, restart:bool = False) -> bool:
logging.info("ATS already running - nothing to do")
return True
- tsexe = os.path.join(TS_ROOT, "bin", "trafficserver")
- if MODE is Modes.INTERACTIVE and not getYN("Run command '%s %s'?" % (tsexe, arg)):
+ tsexe = os.path.join(conf.tsroot, "bin", "trafficserver")
+ if ( conf.mode is Configuration.Modes.INTERACTIVE and
+ not getYN("Run command '%s %s'?" % (tsexe, arg))):
logging.warning("ATS status will not be set - Traffic Ops may not expect this!")
return True
logging.info("ATS status will be set using: %s %s", tsexe, arg)
- if MODE is not Modes.REPORT:
+ if conf.mode is not Configuration.Modes.REPORT:
sub = subprocess.Popen([tsexe, arg], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = sub.communicate()
@@ -225,58 +261,18 @@ def setATSStatus(status:bool, restart:bool = False) -> bool:
return False
return True
-def restartService(service:str) -> bool:
- """
- Restarts a generic systemd service
-
- :param service: The name of the service to be restarted
- :returns: Whether or not the restart was successful
- """
- global HAS_SYSTEMD
- from .utils import getYesNoResponse as getYN
- from .configuration import MODE, Modes
-
-
- if not HAS_SYSTEMD:
- logging.warning("This system doesn't have systemd, services cannot be restarted")
- return True
-
- if MODE is not Modes.REPORT and (MODE is not Modes.INTERACTIVE or
- getYN("Restart %s?" % service, default='Y')):
- logging.info("Restarting %s", service)
- try:
- sub = subprocess.Popen(["systemctl", "restart", service],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- out, err = sub.communicate()
- logging.debug("stdout: %s\nstderr: %s", out, err)
- except (OSError, subprocess.CalledProcessError) as e:
- logging.error("An error occurred when restarting %s: %s", service, e)
- logging.debug("%r", e, exc_info=True, stack_info=True)
- return False
- return True
-
-
-
-def setServiceStatus(chkconfig:dict) -> bool:
+def setServiceStatus(chkconfig:dict, mode:Configuration.Modes) -> bool:
"""
Sets the status of a service based on its 'chkconfig'.
A 'chkconfig' consists of a list of run-levels with either 'on' or 'off' as values.
This allowed specifying what run-levels needed a service. It's now totally deprecated,
but the Traffic Ops back-end doesn't know that yet...
- .. warning:: This function currently ONLY checks for 'trafficserver' chkconfigs.
-
:param chkconfig: A single chkconfig
+ :param mode: The current run-mode
:returns: whether or not the service's status was set successfully
"""
global HAS_SYSTEMD
- from .utils import getYesNoResponse as getYN
- from .configuration import MODE, Modes
-
- if not HAS_SYSTEMD:
- logging.warning("This system doesn't have systemd, services cannot be enabled/disabled")
- return True
try:
status = "enable" if "on" in chkconfig["value"] else "disable"
@@ -286,13 +282,14 @@ def setServiceStatus(chkconfig:dict) -> bool:
logging.debug("%s", e, exc_info=True, stack_info=True)
return False
- if MODE is Modes.INTERACTIVE and not getYN("%s %s?" % (service, status), default='Y'):
+ if (mode is Configuration.Modes.INTERACTIVE and
+ not getYN("%s %s?" % (service, status), default='Y')):
logging.warning("%s will not be %sd - some things may break!", service, status)
return True
logging.info("%s will be %sd", service, status)
- if MODE is not Modes.REPORT:
+ if mode is not Configuration.Modes.REPORT:
try:
sub = subprocess.Popen(["systemctl", status, service],
stdout=subprocess.PIPE,
diff --git a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py
index e9fbecc..dff43cc 100644
--- a/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py
+++ b/infrastructure/cdn-in-a-box/ort/traffic_ops_ort/to_api.py
@@ -23,9 +23,13 @@ It extends the class provided by the official Apache Traffic Control Client.
import typing
import logging
+from requests.exceptions import RequestException
+from requests.compat import urljoin
from trafficops.tosession import TOSession
+from trafficops.restapi import LoginError, OperationError, InvalidJSONError
from . import packaging
+from .configuration import Configuration
class API(TOSession):
"""
@@ -37,33 +41,37 @@ class API(TOSession):
#: older ATC versions. Go figure.
VERSION = "1.4"
- #: Caches update statuses mapped by hostnames
- CACHED_UPDATE_STATUS = {}
-
- def __init__(self, username:str, password:str, toHost:str, myHostname:str, port:int = 443,
- verify:bool = True, useSSL:bool = True):
+ def __init__(self, conf:Configuration):
"""
This not only creates the API session, but log the user in immediately.
- :param username: The name of the user as whom :term:`ORT` will authenticate with Traffic Ops
- :param password: The password of Traffic Ops user ``username``
- :param toHost: The :abbr:`FQDN (Fully Qualified Domain Name)` of the Traffic Ops server
- :param myHostname: The (short) hostname of **this** server
- :param port: The port number on which Traffic Ops listens for incoming HTTP(S) requests
- :param verify: If :const:`True` SSL certificates will be verifed, if :const:`False` they
- will not and warnings about unverified SSL certificates will be swallowed.
- :param useSSL: If :const:`True` :term:`ORT` will attempt to communicate with Traffic Ops
- using SSL, if :const:`False` it will not. *This setting will be respected
- regardless of the passed port number!*
- :raises trafficops.restapi.LoginError: when authentication with Traffic Ops fails
- :raises trafficops.restapi.OperationError: when some anonymous error occurs communicating
- with the Traffic Ops server
+ :param conf: An object that represents the configuration of :program:`traffic_ops_ort`
+ :raises LoginError: when authentication with Traffic Ops fails
+ :raises OperationError: when some anonymous error occurs communicating with the Traffic Ops server
"""
- super(API, self).__init__(host_ip=toHost, api_version=self.VERSION, host_port=port,
- verify_cert=verify, ssl=useSSL)
- self.login(username, password)
+ super(API, self).__init__(host_ip=conf.toHost, api_version=self.VERSION,
+ host_port=conf.toPort, verify_cert=conf.verify, ssl=conf.useSSL)
+
+ self.retries = conf.retries
+
+ for r in range(self.retries):
+ try:
+ logging.info("login attempt #%d", r)
+ self.login(conf.username, conf.password)
+ break
+ except (LoginError, OperationError, InvalidJSONError, RequestException) as e:
+ logging.debug("login failure: %r", e, stack_info=True, exc_info=True)
+ else:
+ raise LoginError("Failed to log in to Traffic Ops, retries exceeded.")
- self.hostname = myHostname
+ self.hostname = conf.shortHostname
+
+ def __enter__(self):
+ """
+ Implements context-management for :class:`API` objects. Needs to override :class:`TOSession`
+ context-management because the connection is already established during initialization.
+ """
+ return self
def getRaw(self, path:str) -> str:
"""
@@ -74,12 +82,20 @@ class API(TOSession):
:param path: The raw path on the Traffic Ops server
:returns: The API response payload
+ :raises ConnectionError: When something goes wrong communicating with the Traffic Ops server
"""
-
- r = self._session.get(path)
+ for _ in range(self.retries):
+ try:
+ r = self._session.get(path)
+ break
+ except (LoginError, OperationError, InvalidJSONError, RequestException) as e:
+ logging.debug("API failure: %r", e, stack_info=True, exc_info=True)
+ else:
+ raise ConnectionError("Failed to get a valid response from Traffic Ops for %s" % path)
if r.status_code != 200 and r.status_code != 204:
- raise ValueError("request for '%s' appears to have failed; reason: %s" % (path, r.reason))
+ raise ConnectionError("request for '%s' appears to have failed; reason: %s" %
+ (path, r.reason))
return r.text
@@ -89,74 +105,101 @@ class API(TOSession):
:returns: all of the packages which this system must have, according to Traffic Ops.
:raises ConnectionError: if fetching the package list fails
- :raises ValueError: if the API endpoint returns a malformed response that can't be parsed
"""
logging.info("Fetching this server's package list from Traffic Ops")
# Ah, read-only properties that gut functionality, my favorite.
- from requests.compat import urljoin
tmp = self.api_base_url
self._api_base_url = urljoin(self._server_url, '/').rstrip('/') + '/'
packagesPath = '/'.join(("ort", self.hostname, "packages"))
- myPackages = self.get(packagesPath)
+ for _ in range(self.retries):
+ try:
+ myPackages = self.get(packagesPath)
+ break
+ except (LoginError, OperationError, InvalidJSONError, RequestException) as e:
+ logging.debug("package fetch failure: %r", e, stack_info=True, exc_info=True)
+ else:
+ self._api_base_url = tmp
+ raise ConnectionError("Failed to get a response for packages")
+
self._api_base_url = tmp
logging.debug("Raw package response: %s", myPackages[1].text)
- return [packaging.Package(p) for p in myPackages[0]]
+ try:
+ return [packaging.Package(p) for p in myPackages[0]]
+ except ValueError:
+ raise ConnectionError
- def getMyConfigFiles(self) -> typing.List[dict]:
+ def getMyConfigFiles(self, conf:Configuration) -> typing.List[dict]:
"""
Fetches configuration files constructed by Traffic Ops for this server
- .. note:: This function will set the :data:`traffic_ops_ort.configuration.SERVER_INFO`
- object to an instance of :class:`ServerInfo` with the provided information.
+ .. note:: This function will set the :attr:`serverInfo` attribute of the object passed as
+ the ``conf`` argument to an instance of :class:`ServerInfo` with the provided
+ information.
+ :param conf: An object that represents the configuration of :program:`traffic_ops_ort`
:returns: A list of constructed config file objects
:raises ConnectionError: when something goes wrong communicating with Traffic Ops
- :raises ValueError: when a response was successfully obtained from the Traffic Ops API, but the
- response could not successfully be parsed as JSON, or was missing information
"""
- from . import configuration
-
logging.info("Fetching list of configuration files from Traffic Ops")
-
- # The API function decorator confuses pylint into thinking this doesn't return
- #pylint: disable=E1111
- myFiles = self.get_server_config_files(host_name=self.hostname)
- #pylint: enable=E1111
+ for _ in range(self.retries):
+ try:
+ # The API function decorator confuses pylint into thinking this doesn't return
+ #pylint: disable=E1111
+ myFiles = self.get_server_config_files(host_name=self.hostname)
+ #pylint: enable=E1111
+ break
+ except (InvalidJSONError, LoginError, OperationError, RequestException) as e:
+ logging.debug("config file fetch failure: %r", e, exc_info=True, stack_info=True)
+ else:
+ raise ConnectionError("Failed to fetch configuration files from Traffic Ops")
logging.debug("Raw response from Traffic Ops: %s", myFiles[1].text)
myFiles = myFiles[0]
try:
- configuration.SERVER_INFO = ServerInfo(myFiles.info)
+ conf.serverInfo = ServerInfo(myFiles.info)
+ # if there's a reverse proxy, switch urls.
+ if conf.serverInfo.toRevProxyUrl and not conf.rev_proxy_disable:
+ self._server_url = conf.serverInfo.toRevProxyUrl
+ self._api_base_url = urljoin(self._server_url, '/api/%s' % self.VERSION).rstrip('/') + '/'
return myFiles.configFiles
- except (KeyError, AttributeError) as e:
- raise ValueError from e
+ except (KeyError, AttributeError, ValueError) as e:
+ raise ConnectionError("Malformed response from Traffic Ops to update status request!") from e
- def updateTrafficOps(self):
+ def updateTrafficOps(self, mode:Configuration.Modes):
"""
Updates Traffic Ops's knowledge of this server's update status.
+
+ :param mode: The current run-mode of :program:`traffic_ops_ort`
"""
- from .configuration import MODE, Modes
from .utils import getYesNoResponse as getYN
- if MODE is Modes.INTERACTIVE and not getYN("Update Traffic Ops?", default='Y'):
+ if mode is Configuration.Modes.INTERACTIVE and not getYN("Update Traffic Ops?", default='Y'):
logging.warning("Update will not be performed; you should clear updates manually")
return
logging.info("Updating Traffic Ops")
- if MODE is Modes.REPORT:
+ if mode is Configuration.Modes.REPORT:
return
payload = {"updated": False, "reval_updated": False}
- response = self._session.post('/'.join((self._server_url.rstrip('/'),
- "update",
- self.hostname)
- ), data=payload)
+
+ for _ in range(self.retries):
+ try:
+ response = self._session.post('/'.join((self._server_url.rstrip('/'),
+ "update",
+ self.hostname)
+ ), data=payload)
+ break
+ except (LoginError, InvalidJSONError, OperationError, RequestException) as e:
+ logging.debug("TO update failure: %r", e, exc_info=True, stack_info=True)
+ else:
+ raise ConnectionError("Failed to update Traffic Ops - connection was lost")
if response.text:
logging.info("Traffic Ops response: %s", response.text)
@@ -167,50 +210,54 @@ class API(TOSession):
:returns: An iterable list of 'chkconfig' entries
:raises ConnectionError: when something goes wrong communicating with Traffic Ops
- :raises ValueError: when a response was successfully obtained from the Traffic Ops API, but the
- response could not successfully be parsed as JSON, or was missing information
"""
# Ah, read-only properties that gut functionality, my favorite.
- from requests.compat import urljoin
tmp = self.api_base_url
self._api_base_url = urljoin(self._server_url, '/').rstrip('/') + '/'
uri = "ort/%s/chkconfig" % self.hostname
logging.info("Fetching chkconfig from %s", uri)
- r = self.get(uri)
+ for _ in range(self.retries):
+ try:
+ r = self.get(uri)
+ break
+ except (InvalidJSONError, OperationError, LoginError, RequestException) as e:
+ logging.debug("chkconfig fetch failure: %r", e, exc_info=True, stack_info=True)
+ else:
+ self._api_base_url = tmp
+ raise ConnectionError("Failed to fetch 'chkconfig' from Traffic Ops - connection lost")
+
self._api_base_url = tmp
+
logging.debug("Raw response from Traffic Ops: %s", r[1].text)
return r[0]
- def getUpdateStatus(self, host:str) -> dict:
+ def getMyUpdateStatus(self) -> dict:
"""
Gets the update status of a server.
- .. note:: If the :data:`self.CACHED_UPDATE_STATUS` cached response is set, this function will
- default to that object. If it is *not* set, then this function will set it.
-
- :param host: The (short) hostname of the server to query
- :raises PermissionError: if a new cookie is required, but fails to be aquired
+ :raises ConnectionError: if something goes wrong communicating with the server
:returns: An object representing the API's response
"""
+ logging.info("Fetching update status from Traffic Ops")
+ for _ in range(self.retries):
+ try:
+ # The API function decorator confuses pylint into thinking this doesn't return
+ #pylint: disable=E1111
+ r = self.get_server_update_status(server_name=self.hostname)
+ #pylint: enable=E1111
+ break
+ except (InvalidJSONError, LoginError, OperationError, RequestException) as e:
+ logging.debug("update status fetch failure: %r", e, exc_info=True, stack_info=True)
+ else:
+ raise ConnectionError("Failed to fetch update status - connection was lost")
- logging.info("Fetching update status for %s from Traffic Ops", host)
-
- if host in self.CACHED_UPDATE_STATUS:
- return self.CACHED_UPDATE_STATUS[host]
-
- # The API function decorator confuses pylint into thinking this doesn't return
- #pylint: disable=E1111
- r = self.get_server_update_status(server_name=host)
- #pylint: enable=E1111
logging.debug("Raw response from Traffic Ops: %s", r[1].text)
- self.CACHED_UPDATE_STATUS[host] = r[0]
-
return r[0]
def getMyStatus(self) -> str:
@@ -218,21 +265,22 @@ class API(TOSession):
Fetches the status of this server as set in Traffic Ops
:raises ConnectionError: if fetching the status fails
- :raises ValueError: if the :data:`traffic_ops_ort.configuration.HOSTNAME` is not properly set,
- or a weird value is stored in the global :data:`CACHED_UPDATE_STATUS` response cache.
:returns: the name of the status to which this server is set in the Traffic Ops configuration
- .. note:: If the global :data:`CACHED_UPDATE_STATUS` cached response is set, this function will
- default to the status provided by that object.
"""
-
-
logging.info("Fetching server status from Traffic Ops")
- # The API function decorator confuses pylint into thinking this doesn't return
- #pylint: disable=E1111
- r = self.get_servers(query_params={"hostName": self.hostname})
- #pylint: enable=E1111
+ for _ in range(self.retries):
+ try:
+ # The API function decorator confuses pylint into thinking this doesn't return
+ #pylint: disable=E1111
+ r = self.get_servers(query_params={"hostName": self.hostname})
+ #pylint: enable=E1111
+ break
+ except (InvalidJSONError, LoginError, OperationError,RequestException) as e:
+ logging.debug("status fetch failure: %r", e, exc_info=True, stack_info=True)
+ else:
+ raise ConnectionError("Failed to fetch server status - connection was lost")
logging.debug("Raw response from Traffic Ops: %s", r[1].text)
@@ -241,8 +289,7 @@ class API(TOSession):
try:
return r.status
except (IndexError, KeyError, AttributeError) as e:
- logging.error("Malformed response from Traffic Ops to update status request!")
- raise ConnectionError from e
+ raise ConnectionError("Malformed response from Traffic Ops to update status request!") from e
#: Caches the names of statuses supported by Traffic Ops
CACHED_STATUSES = []
@@ -309,13 +356,16 @@ class ServerInfo():
for a in dir(self)\
if not a.startswith('_')))
- def sanitize(self, fmt:str) -> str:
+ def sanitize(self, fmt:str, hostname:typing.Tuple[str, str]) -> str:
"""
- Implements ``str.format(self)``
+ Sanitizes an input string with the passed hostname information
+
+ :param fmt: The string to be sanitized
+ :param hostname: A tuple containing the short and full hostnames of the server
+ :returns: The string ``fmt`` after sanitization
"""
- from .configuration import HOSTNAME
- fmt = fmt.replace("__HOSTNAME__", HOSTNAME[0])
- fmt = fmt.replace("__FULL_HOSTNAME__", HOSTNAME[1])
+ fmt = fmt.replace("__HOSTNAME__", hostname[0])
+ fmt = fmt.replace("__FULL_HOSTNAME__", hostname[1])
fmt = fmt.replace("__RETURN__", '\n')
fmt = fmt.replace("__CACHE_IPV4__", self.serverIpv4)