You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafficcontrol.apache.org by oc...@apache.org on 2023/07/26 20:14:06 UTC
[trafficcontrol] branch master updated: t3c to use package metadata if rpm db is unhealthy (#7652)
This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new 2233d22ed2 t3c to use package metadata if rpm db is unhealthy (#7652)
2233d22ed2 is described below
commit 2233d22ed2d199869a98d50b9fd9bd1f54fa3fab
Author: Joe Pappano <jo...@cable.comcast.com>
AuthorDate: Wed Jul 26 16:13:59 2023 -0400
t3c to use package metadata if rpm db is unhealthy (#7652)
* adding check to verify rpmdb
* added check to verify rpm db
* rebase and fix conflicts
* if rpmdb is unhealthy use package data in metadata
* requested changes addressed.
* added changelog entry
* fixed spelling error and added GoDoc commnet
* fixed formatting error
---
CHANGELOG.md | 1 +
cache-config/t3c-apply/config/config.go | 69 ++++++++++++++++++++++-----
cache-config/t3c-apply/t3c-apply.go | 10 ++--
cache-config/t3c-apply/torequest/torequest.go | 47 +++++++++++++++++-
4 files changed, 110 insertions(+), 17 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 55f97ce1e2..134e3eabe9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
- [#7619](https://github.com/apache/trafficcontrol/pull/7619) Traffic Ops* added optional field `oauth_user_attribute` for OAuth login credentials
- [#7641](https://github.com/apache/trafficcontrol/pull/7641) *Traffic Router* Added further optimization to TR's algorithm of figuring out the zone for an incoming request.
- [#7646](https://github.com/apache/trafficcontrol/pull/7646) *Traffic Portal* Add the ability to delete a cert.
+- [#7652](https://github.com/apache/trafficcontrol/pull/7652) *t3c* added rpmdb checks and use package data from t3c-apply-metadata.json if rpmdb is corrupt
### Changed
- [#7584](https://github.com/apache/trafficcontrol/pull/7584) *Documentation* Upgrade Traffic Control Sphinx documentation Makefile OS intelligent.
diff --git a/cache-config/t3c-apply/config/config.go b/cache-config/t3c-apply/config/config.go
index 64a62a0160..e7956e1eba 100644
--- a/cache-config/t3c-apply/config/config.go
+++ b/cache-config/t3c-apply/config/config.go
@@ -27,6 +27,7 @@ import (
"os"
"os/exec"
"path/filepath"
+ "regexp"
"strings"
"time"
@@ -81,6 +82,7 @@ type Cfg struct {
SvcManagement SvcManagement
Retries int
ReverseProxyDisable bool
+ RpmDBOk bool
SkipOSCheck bool
UseStrategies t3cutil.UseStrategiesFlag
TOInsecure bool
@@ -188,6 +190,29 @@ func directoryExists(dir string) (bool, os.FileInfo) {
return info.IsDir(), info
}
+const rpmDir = "/var/lib/rpm"
+
+// verifies the rpm database files. if there is any database corruption
+// it will return false
+func verifyRpmDB() bool {
+ exclude := regexp.MustCompile(`(^\.|^__)`)
+ dbFiles, err := os.ReadDir(rpmDir)
+ if err != nil {
+ return false
+ }
+ for _, file := range dbFiles {
+ if exclude.Match([]byte(file.Name())) {
+ continue
+ }
+ cmd := exec.Command("/usr/lib/rpm/rpmdb_verify", rpmDir+"/"+file.Name())
+ err := cmd.Run()
+ if err != nil || cmd.ProcessState.ExitCode() > 0 {
+ return false
+ }
+ }
+ return true
+}
+
// derives the ATS Installation directory from
// the rpm config file list.
func GetTSPackageHome() string {
@@ -322,10 +347,11 @@ If any of the related flags are also set, they override the mode's default behav
// so we want to log what flags the mode set here, to aid debugging.
// But we can't do that until the loggers are initialized.
modeLogStrs := []string{}
+ fatalLogStrs := []string{}
if getopt.IsSet(runModeFlagName) {
runMode := t3cutil.StrToMode(*runModePtr)
if runMode == t3cutil.ModeInvalid {
- return Cfg{}, errors.New(*runModePtr + " is an invalid mode.")
+ fatalLogStrs = append(fatalLogStrs, *runModePtr+" is an invalid mode.")
}
modeLogStrs = append(modeLogStrs, "t3c-apply is running in "+runMode.String()+" mode")
switch runMode {
@@ -411,7 +437,7 @@ If any of the related flags are also set, they override the mode's default behav
}
if *verbosePtr > 2 {
- return Cfg{}, errors.New("Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)")
+ fatalLogStrs = append(fatalLogStrs, "Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)")
}
var cacheHostName string
@@ -420,7 +446,7 @@ If any of the related flags are also set, they override the mode's default behav
} else {
cacheHostName, err = os.Hostname()
if err != nil {
- return Cfg{}, errors.New("Could not get the hostname from the O.S., please supply a hostname: " + err.Error())
+ fatalLogStrs = append(fatalLogStrs, "Could not get the hostname from the O.S., please supply a hostname: "+err.Error())
}
// strings.Split always returns a slice with at least 1 element, so we don't need a len check
cacheHostName = strings.Split(cacheHostName, ".")[0]
@@ -429,7 +455,7 @@ If any of the related flags are also set, they override the mode's default behav
useGit := StrToUseGitFlag(*useGitStr)
if useGit == UseGitInvalid {
- return Cfg{}, errors.New("Invalid git flag '" + *useGitStr + "'. Valid options are yes, no, auto.")
+ fatalLogStrs = append(fatalLogStrs, "Invalid git flag '"+*useGitStr+"'. Valid options are yes, no, auto.")
}
retries := *retriesPtr
@@ -471,6 +497,17 @@ If any of the related flags are also set, they override the mode's default behav
os.Setenv("TO_PASS", toPass)
}
+ rpmDBisOk := verifyRpmDB()
+
+ if *installPackagesPtr && !rpmDBisOk {
+ if t3cutil.StrToMode(*runModePtr) == t3cutil.ModeBadAss {
+ fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue in badass mode")
+ } else {
+ fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue")
+ }
+ }
+
+ toInfoLog = append(toInfoLog, fmt.Sprintf("rpm database is ok: %t", rpmDBisOk))
// set TSHome
var tsHome = ""
if *tsHomePtr != "" {
@@ -481,13 +518,13 @@ If any of the related flags are also set, they override the mode's default behav
tsHome = os.Getenv("TS_HOME") // check for the environment variable.
if tsHome != "" {
toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from TS_HOME environment variable '%s'\n", TSHome))
- } else { // finally check using the config file listing from the rpm package.
+ } else if rpmDBisOk { // check using the config file listing from the rpm package if rpmdb is ok.
tsHome = GetTSPackageHome()
if tsHome != "" {
toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from the RPM config file list '%s'\n", TSHome))
- } else {
- toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome))
}
+ } else if tsHome == "" {
+ toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome))
}
}
@@ -503,23 +540,23 @@ If any of the related flags are also set, they override the mode's default behav
if *useLocalATSVersionPtr {
atsVersionStr, err = GetATSVersionStr(tsHome)
if err != nil {
- return Cfg{}, errors.New("getting local ATS version: " + err.Error())
+ fatalLogStrs = append(fatalLogStrs, "getting local ATS version: "+err.Error())
}
}
toInfoLog = append(toInfoLog, fmt.Sprintf("ATSVersionStr: '%s'\n", atsVersionStr))
usageStr := "basic usage: t3c-apply --traffic-ops-url=myurl --traffic-ops-user=myuser --traffic-ops-password=mypass --cache-host-name=my-cache"
if strings.TrimSpace(toURL) == "" {
- return Cfg{}, errors.New("Missing required argument --traffic-ops-url or TO_URL environment variable. " + usageStr)
+ fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-url or TO_URL environment variable. "+usageStr)
}
if strings.TrimSpace(toUser) == "" {
- return Cfg{}, errors.New("Missing required argument --traffic-ops-user or TO_USER environment variable. " + usageStr)
+ fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-user or TO_USER environment variable. "+usageStr)
}
if strings.TrimSpace(toPass) == "" {
- return Cfg{}, errors.New("Missing required argument --traffic-ops-password or TO_PASS environment variable. " + usageStr)
+ fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-password or TO_PASS environment variable. "+usageStr)
}
if strings.TrimSpace(cacheHostName) == "" {
- return Cfg{}, errors.New("Missing required argument --cache-host-name. " + usageStr)
+ fatalLogStrs = append(fatalLogStrs, "Missing required argument --cache-host-name. "+usageStr)
}
toURLParsed, err := url.Parse(toURL)
@@ -540,6 +577,7 @@ If any of the related flags are also set, they override the mode's default behav
CacheHostName: cacheHostName,
SvcManagement: svcManagement,
Retries: retries,
+ RpmDBOk: rpmDBisOk,
ReverseProxyDisable: reverseProxyDisable,
SkipOSCheck: skipOsCheck,
UseStrategies: useStrategies,
@@ -580,6 +618,13 @@ If any of the related flags are also set, they override the mode's default behav
return Cfg{}, errors.New("Initializing loggers: " + err.Error() + "\n")
}
+ if len(fatalLogStrs) > 0 {
+ for _, str := range fatalLogStrs {
+ str = strings.TrimSpace(str)
+ log.Errorln(str)
+ }
+ return Cfg{}, errors.New("fatal error has occurred")
+ }
for _, str := range modeLogStrs {
str = strings.TrimSpace(str)
if str == "" {
diff --git a/cache-config/t3c-apply/t3c-apply.go b/cache-config/t3c-apply/t3c-apply.go
index bc98fcf23a..24e1e8e91a 100644
--- a/cache-config/t3c-apply/t3c-apply.go
+++ b/cache-config/t3c-apply/t3c-apply.go
@@ -22,7 +22,6 @@ package main
import (
"encoding/json"
"errors"
- "fmt"
"io/ioutil"
"os"
"path/filepath"
@@ -94,8 +93,8 @@ func Main() int {
var lock util.FileLock
cfg, err := config.GetCfg(Version, GitRevision)
if err != nil {
- fmt.Println(err)
- fmt.Println(FailureExitMsg)
+ log.Infoln(err)
+ log.Errorln(FailureExitMsg)
return ExitCodeConfigError
} else if cfg == (config.Cfg{}) { // user used the --help option
return ExitCodeSuccess
@@ -261,7 +260,7 @@ func Main() int {
// make sure we got the data necessary to check packages
log.Infoln("======== Didn't get all files, no package processing needed or possible ========")
metaData.InstalledPackages = oldMetaData.InstalledPackages
- } else {
+ } else if cfg.RpmDBOk {
log.Infoln("======== Start processing packages ========")
err = trops.ProcessPackages()
if err != nil {
@@ -276,6 +275,9 @@ func Main() int {
log.Errorf("Error verifying system services: %s\n", err.Error())
return GitCommitAndExit(ExitCodeServicesError, FailureExitMsg, cfg, metaData, oldMetaData)
}
+ } else {
+ log.Warnln("======== RPM DB checks failed, package processing not possible, using installed packages from metadata if available========")
+ trops.ProcessPackagesWithMetaData(oldMetaData.InstalledPackages)
}
log.Debugf("Preparing to fetch the config files for %s, files: %s, syncdsUpdate: %s\n", cfg.CacheHostName, cfg.Files, syncdsUpdate)
diff --git a/cache-config/t3c-apply/torequest/torequest.go b/cache-config/t3c-apply/torequest/torequest.go
index 1b6016abf1..5298091f6a 100644
--- a/cache-config/t3c-apply/torequest/torequest.go
+++ b/cache-config/t3c-apply/torequest/torequest.go
@@ -587,10 +587,14 @@ func (r *TrafficOpsReq) CheckSystemServices() error {
func (r *TrafficOpsReq) IsPackageInstalled(name string) bool {
for k, v := range r.Pkgs {
if strings.HasPrefix(k, name) {
+ log.Infof("Found in cache for '%s'", k)
return v
}
}
-
+ if !r.Cfg.RpmDBOk {
+ log.Warnf("RPM DB is corrupted cannot run IsPackageInstalled for '%s' and package metadata is unavailable", name)
+ return false
+ }
log.Infof("IsPackageInstalled '%v' not found in cache, querying rpm", name)
pkgArr, err := util.PackageInfo("pkg-query", name)
if err != nil {
@@ -1030,6 +1034,47 @@ func (r *TrafficOpsReq) ProcessPackages() error {
return nil
}
+func pkgMetaDataToMap(pmd []string) map[string]bool {
+ pkgMap := map[string]bool{}
+ for _, pkg := range pmd {
+ pkgMap[pkg] = true
+ }
+ return pkgMap
+}
+
+func pkgMatch(pkgMetaData []string, pk string) bool {
+ for _, pkg := range pkgMetaData {
+ if strings.Contains(pk, pkg) {
+ return true
+ }
+ }
+ return false
+
+}
+
+// ProcessPackagesWithMetaData will attempt to get installed package data from
+// t3c-apply-metadata.json and log the results.
+func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) error {
+ pkgs, err := getPackages(r.Cfg)
+ pkgMdataMap := pkgMetaDataToMap(packageMetaData)
+ if err != nil {
+ return fmt.Errorf("getting packages: %w", err)
+ }
+ for _, pkg := range pkgs {
+ fullPackage := pkg.Name + "-" + pkg.Version
+ if pkgMdataMap[fullPackage] {
+ log.Infof("package %s is assumed to be installed according to metadata file", fullPackage)
+ r.Pkgs[fullPackage] = true
+ } else if pkgMatch(packageMetaData, pkg.Name) {
+ log.Infof("package %s is assumed to be installed according to metadata, but doesn't match traffic ops pkg", fullPackage)
+ r.Pkgs[fullPackage] = true
+ } else {
+ log.Infof("package %s does not appear to be installed.", pkg.Name+"-"+pkg.Version)
+ }
+ }
+ return nil
+}
+
func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
updateStatus, err := r.CheckRevalidateState(true)
if err != nil {