You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@trafficcontrol.apache.org by GitBox <gi...@apache.org> on 2021/06/09 23:52:33 UTC
[GitHub] [trafficcontrol] rawlinp commented on a change in pull request #5924: Add tool to migrate data between TV backends
rawlinp commented on a change in pull request #5924:
URL: https://github.com/apache/trafficcontrol/pull/5924#discussion_r648709675
##########
File path: lib/go-util/encrypt.go
##########
@@ -0,0 +1,77 @@
+package util
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/rand"
+ "io"
+)
+
+// AESEncrypt takes in a 16, 24, or 32 byte AES key (128, 192, 256 bit encryption relatively) and plain text. It
Review comment:
I think you mean "respectively" instead of "relatively"
##########
File path: tools/traffic_vault_migrate/traffic_vault_migrate.go
##########
@@ -0,0 +1,448 @@
+package main
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "reflect"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+var (
+ fromType string
+ toType string
+ fromCfg string
+ toCfg string
+ dry bool
+ compare bool
+ confirm bool
+ dump bool
+
+ riakBE RiakBackend = RiakBackend{}
+ pgBE PGBackend = PGBackend{}
+)
+
+func init() {
+ flag.StringVar(&fromType, "from_type", riakBE.Name(), fmt.Sprintf("From server types (%v)", strings.Join(supportedTypes(), "|")))
+ flag.StringVar(&toType, "to_type", pgBE.Name(), fmt.Sprintf("From server types (%v)", strings.Join(supportedTypes(), "|")))
+ flag.StringVar(&toCfg, "to_cfg", "pg.json", "To server config file")
+ flag.StringVar(&fromCfg, "from_cfg", "riak.json", "From server config file")
+ flag.BoolVar(&dry, "dry", false, "Do not perform writes")
+ flag.BoolVar(&compare, "compare", false, "Compare to and from server records")
+ flag.BoolVar(&confirm, "confirm", true, "Requires confirmation before inserting records")
+ flag.BoolVar(&dump, "dump", false, "Write keys (from 'from' server) to disk")
+}
+
+func supportedBackends() []TVBackend {
+ return []TVBackend{
+ &riakBE, &pgBE,
+ }
+}
+
+func main() {
+ flag.Parse()
+ var fromSrv TVBackend
+ var toSrv TVBackend
+
+ toSrvUsed := !dump && !dry
+
+ if !validateType(fromType) {
+ log.Fatal("Unknown fromType " + fromType)
+ }
+ if toSrvUsed && !validateType(toType) {
+ log.Fatal("Unknown toType " + toType)
+ }
+
+ fromSrv = getBackendFromType(fromType)
+ if toSrvUsed {
+ toSrv = getBackendFromType(toType)
+ }
+
+ var toTimer time.Time
+ var toTime float64
+ var fromTimer time.Time
+ var fromTime float64
+
+ log.Println("Reading configs...")
+ fromTimer = time.Now()
+ if err := fromSrv.ReadConfig(fromCfg); err != nil {
+ log.Fatalf("Unable to read fromSrv cfg: %v", err)
+ }
+ fromTime = time.Now().Sub(fromTimer).Seconds()
+
+ if toSrvUsed {
+ toTimer = time.Now()
+ if err := toSrv.ReadConfig(toCfg); err != nil {
+ log.Fatalf("Unable to read toSrv cfg: %v", err)
+ }
+ toTime := time.Now().Sub(toTimer).Seconds()
+ log.Printf("Done [%v seconds]\n\tto: [%v seconds]\n\tfrom: [%v seconds]\n", toTime+fromTime, toTime, fromTime)
+ } else {
+ log.Printf("Done [%v seconds]\n", fromTime)
+ }
+
+ log.Println("Starting servers...")
+ fromTimer = time.Now()
+ if err := fromSrv.Start(); err != nil {
+ log.Fatalf("issue starting fromSrv: %v", err)
+ }
+ fromTime = time.Now().Sub(fromTimer).Seconds()
+ defer func() {
+ fromSrv.Stop()
+ }()
+ if toSrvUsed {
+ toTimer = time.Now()
+ if err := toSrv.Start(); err != nil {
+ log.Fatalf("issue starting toSrv: %v", err)
+ }
+ toTime = time.Now().Sub(toTimer).Seconds()
+ defer func() {
+ toSrv.Stop()
+ }()
+ log.Printf("Done [%v seconds]\n\tto: [%v seconds]\n\tfrom: [%v seconds]\n", toTime+fromTime, toTime, fromTime)
+ } else {
+ log.Printf("Done [%v seconds]\n", fromTime)
+ }
+
+ log.Println("Pinging servers...")
+ fromTimer = time.Now()
+ if err := fromSrv.Ping(); err != nil {
+ log.Fatalf("Unable to ping fromSrv: %v", err)
+ }
+ fromTime = time.Now().Sub(fromTimer).Seconds()
+ if toSrvUsed {
+ toTimer = time.Now()
+ if err := toSrv.Ping(); err != nil {
+ log.Fatalf("Unable to ping toSrv: %v", err)
+ }
+ toTime = time.Now().Sub(toTimer).Seconds()
+ log.Printf("Done [%v seconds]\n\tto: [%v seconds]\n\tfrom: [%v seconds]\n", toTime+fromTime, toTime, fromTime)
+ } else {
+ log.Printf("Done [%v seconds]\n", fromTime)
+ }
+
+ log.Printf("Fetching data from %v...\n", fromSrv.Name())
+ fromTimer = time.Now()
+ if err := fromSrv.Fetch(); err != nil {
+ log.Fatalf("Unable to fetch fromSrv data: %v", err)
+ }
+
+ fromSSLKeys, fromDNSSecKeys, fromURIKeys, fromURLKeys, err := GetKeys(fromSrv)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ if err := Validate(fromSrv); err != nil {
+ log.Fatal(err)
+ }
+ log.Printf("Done [%v seconds]\n", time.Now().Sub(fromTimer).Seconds())
+
+ if dump {
+ log.Printf("Dumping data from %v...\n", fromSrv.Name())
+ fromTimer = time.Now()
+ sslKeysBytes, err := json.Marshal(&fromSSLKeys)
Review comment:
There's a pattern here -- we're doing the same thing 4 times in a row: marshalling keys and writing to a file. Can we condense this into a loop that iterates over a slice of structs to marshal the json and write to disk?
##########
File path: tools/traffic_vault_migrate/riak.json
##########
@@ -0,0 +1,8 @@
+{
+ "user": "admin",
+ "password": "riakAdmin",
+ "host": "localhost",
+ "port": "8087",
+ "tls": false,
Review comment:
RE: earlier comment, this should be `insecure` instead of just `tls`
##########
File path: docs/source/tools/traffic_vault_migrate.rst
##########
@@ -0,0 +1,107 @@
+..
+..
+.. Licensed under the Apache License, Version 2.0 (the "License");
+.. you may not use this file except in compliance with the License.
+.. You may obtain a copy of the License at
+..
+.. http://www.apache.org/licenses/LICENSE-2.0
+..
+.. Unless required by applicable law or agreed to in writing, software
+.. distributed under the License is distributed on an "AS IS" BASIS,
+.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+.. See the License for the specific language governing permissions and
+.. limitations under the License.
+..
+
+.. _traffic_vault_migrate:
+
+#########################
+Traffic Vault Migrate
+#########################
+The ``traffic_vault_migrate`` tool - located at :file:`tools/traffic_vault_migrate/traffic_vault_migrate.go` in the `Apache Traffic Control repository <https://github.com/apache/trafficcontrol>`_ -
+is used to transfer TV keys between database servers. It interfaces directly with each backend so Traffic Ops/Vault being available is not a requirement.
+The tool assumes that the schema for each backend is already setup as according to the :ref:`admin setup <traffic_vault_admin>`.
+
+.. program:: traffic_vault_migrate
+
+Usage
+===========
+``traffic_vault_migrate -from_cfg CFG -to_cfg CFG -from_type TYP -to_type TYP [-confirm] [-compare] [-dry] [-dump]``
+
+.. option:: -compare
+
+ Compare 'to' and 'from' backend keys. Will fetch keys from the dbs of both 'to' and 'from', sorts them by cdn/ds/version and does a deep comparison.
+
+.. option:: -confirm
+
+ Requires confirmation before inserting records (default true)
+
+.. option:: -dry
+
+ Do not perform writes. Will do a basic output of the keys on the 'from' backend.
+
+.. option:: -dump
+
+ Write keys (from 'from' server) to disk in the folder 'dump' with the unix permissions 0640.
+
+ .. warning:: This can write potentially sensitive information to disk, use with care.
+
+.. option:: -from_cfg
+
+ From server config file (default "riak.json")
+
+.. option:: -from_type
+
+ From server types (Riak|PG) (default "Riak")
+
+.. option:: -to_cfg
+
+ To server config file (default "pg.json")
+
+.. option:: -to_type
+
+ From server types (Riak|PG) (default "PG")
+
+Riak
+----------
+
+riak.json
+""""""""""
+
+ :user: The username used to log into the Riak server.
+
+ :password: The password used to log into the Riak server.
+
+ :host: The hostname for the Riak server.
+
+ :port: The port for which the Riak server is listening for protobuf connections.
+
+ :tls: (Optional) Determines whether to verify insecure certificates.
Review comment:
this should probably be `insecure` (default = false) instead of just `tls`
##########
File path: lib/go-util/encrypt.go
##########
@@ -0,0 +1,77 @@
+package util
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/rand"
+ "io"
+)
+
+// AESEncrypt takes in a 16, 24, or 32 byte AES key (128, 192, 256 bit encryption relatively) and plain text. It
+// returns the encrypted text. In case of error, the text returned is an empty string. AES requires input text to
+// be greater than 12 bytes in length.
+func AESEncrypt(bytesToEncrypt []byte, aesKey []byte) ([]byte, error) {
+ cipherBlock, err := aes.NewCipher(aesKey)
+ if err != nil {
+ return []byte{}, err
+ }
+
+ gcm, err := cipher.NewGCM(cipherBlock)
+ if err != nil {
+ return []byte{}, err
+ }
+
+ nonce := make([]byte, gcm.NonceSize())
+ if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+ return []byte{}, err
+ }
+
+ return gcm.Seal(nonce, nonce, bytesToEncrypt, nil), nil
+}
+
+// AESDecrypt takes in a 16, 24, or 32 byte AES key (128, 192, 256 bit encryption relatively) and encrypted text. It
Review comment:
I think you mean "respectively" instead of "relatively"
##########
File path: tools/traffic_vault_migrate/traffic_vault_migrate.go
##########
@@ -0,0 +1,448 @@
+package main
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "reflect"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+var (
+ fromType string
+ toType string
+ fromCfg string
+ toCfg string
+ dry bool
+ compare bool
+ confirm bool
+ dump bool
+
+ riakBE RiakBackend = RiakBackend{}
+ pgBE PGBackend = PGBackend{}
+)
+
+func init() {
+ flag.StringVar(&fromType, "from_type", riakBE.Name(), fmt.Sprintf("From server types (%v)", strings.Join(supportedTypes(), "|")))
+ flag.StringVar(&toType, "to_type", pgBE.Name(), fmt.Sprintf("From server types (%v)", strings.Join(supportedTypes(), "|")))
+ flag.StringVar(&toCfg, "to_cfg", "pg.json", "To server config file")
+ flag.StringVar(&fromCfg, "from_cfg", "riak.json", "From server config file")
+ flag.BoolVar(&dry, "dry", false, "Do not perform writes")
+ flag.BoolVar(&compare, "compare", false, "Compare to and from server records")
+ flag.BoolVar(&confirm, "confirm", true, "Requires confirmation before inserting records")
+ flag.BoolVar(&dump, "dump", false, "Write keys (from 'from' server) to disk")
+}
+
+func supportedBackends() []TVBackend {
+ return []TVBackend{
+ &riakBE, &pgBE,
+ }
+}
+
+func main() {
+ flag.Parse()
+ var fromSrv TVBackend
+ var toSrv TVBackend
+
+ toSrvUsed := !dump && !dry
+
+ if !validateType(fromType) {
+ log.Fatal("Unknown fromType " + fromType)
+ }
+ if toSrvUsed && !validateType(toType) {
+ log.Fatal("Unknown toType " + toType)
+ }
+
+ fromSrv = getBackendFromType(fromType)
+ if toSrvUsed {
+ toSrv = getBackendFromType(toType)
+ }
+
+ var toTimer time.Time
+ var toTime float64
+ var fromTimer time.Time
+ var fromTime float64
+
+ log.Println("Reading configs...")
+ fromTimer = time.Now()
+ if err := fromSrv.ReadConfig(fromCfg); err != nil {
+ log.Fatalf("Unable to read fromSrv cfg: %v", err)
+ }
+ fromTime = time.Now().Sub(fromTimer).Seconds()
+
+ if toSrvUsed {
+ toTimer = time.Now()
+ if err := toSrv.ReadConfig(toCfg); err != nil {
+ log.Fatalf("Unable to read toSrv cfg: %v", err)
+ }
+ toTime := time.Now().Sub(toTimer).Seconds()
+ log.Printf("Done [%v seconds]\n\tto: [%v seconds]\n\tfrom: [%v seconds]\n", toTime+fromTime, toTime, fromTime)
+ } else {
+ log.Printf("Done [%v seconds]\n", fromTime)
+ }
+
+ log.Println("Starting servers...")
+ fromTimer = time.Now()
+ if err := fromSrv.Start(); err != nil {
+ log.Fatalf("issue starting fromSrv: %v", err)
+ }
+ fromTime = time.Now().Sub(fromTimer).Seconds()
+ defer func() {
+ fromSrv.Stop()
+ }()
+ if toSrvUsed {
+ toTimer = time.Now()
+ if err := toSrv.Start(); err != nil {
+ log.Fatalf("issue starting toSrv: %v", err)
+ }
+ toTime = time.Now().Sub(toTimer).Seconds()
+ defer func() {
+ toSrv.Stop()
+ }()
+ log.Printf("Done [%v seconds]\n\tto: [%v seconds]\n\tfrom: [%v seconds]\n", toTime+fromTime, toTime, fromTime)
+ } else {
+ log.Printf("Done [%v seconds]\n", fromTime)
+ }
+
+ log.Println("Pinging servers...")
+ fromTimer = time.Now()
+ if err := fromSrv.Ping(); err != nil {
+ log.Fatalf("Unable to ping fromSrv: %v", err)
+ }
+ fromTime = time.Now().Sub(fromTimer).Seconds()
+ if toSrvUsed {
+ toTimer = time.Now()
+ if err := toSrv.Ping(); err != nil {
+ log.Fatalf("Unable to ping toSrv: %v", err)
+ }
+ toTime = time.Now().Sub(toTimer).Seconds()
+ log.Printf("Done [%v seconds]\n\tto: [%v seconds]\n\tfrom: [%v seconds]\n", toTime+fromTime, toTime, fromTime)
+ } else {
+ log.Printf("Done [%v seconds]\n", fromTime)
+ }
+
+ log.Printf("Fetching data from %v...\n", fromSrv.Name())
+ fromTimer = time.Now()
+ if err := fromSrv.Fetch(); err != nil {
+ log.Fatalf("Unable to fetch fromSrv data: %v", err)
+ }
+
+ fromSSLKeys, fromDNSSecKeys, fromURIKeys, fromURLKeys, err := GetKeys(fromSrv)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ if err := Validate(fromSrv); err != nil {
+ log.Fatal(err)
+ }
+ log.Printf("Done [%v seconds]\n", time.Now().Sub(fromTimer).Seconds())
+
+ if dump {
+ log.Printf("Dumping data from %v...\n", fromSrv.Name())
+ fromTimer = time.Now()
+ sslKeysBytes, err := json.Marshal(&fromSSLKeys)
+ if err != nil {
+ log.Fatal(err)
+ }
+ dnssecKeyBytes, err := json.Marshal(&fromDNSSecKeys)
+ if err != nil {
+ log.Fatal(err)
+ }
+ uriKeyBytes, err := json.Marshal(&fromURIKeys)
+ if err != nil {
+ log.Fatal(err)
+ }
+ urlKeyBytes, err := json.Marshal(&fromURLKeys)
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ if err := os.Mkdir("dump", 0750); err != nil {
+ if !os.IsExist(err) {
+ log.Fatal(err)
+ }
+ }
+ if err = ioutil.WriteFile("dump/sslkeys.json", sslKeysBytes, 0640); err != nil {
+ log.Println(err)
+ }
+ if err = ioutil.WriteFile("dump/dnsseckeys.json", dnssecKeyBytes, 0640); err != nil {
+ log.Println(err)
+ }
+ if err = ioutil.WriteFile("dump/urlkeys.json", urlKeyBytes, 0640); err != nil {
+ log.Println(err)
+ }
+ if err = ioutil.WriteFile("dump/urikeys.json", uriKeyBytes, 0640); err != nil {
+ log.Println(err)
+ }
+ log.Printf("Done [%v seconds]\n", time.Now().Sub(fromTimer).Seconds())
+ return
+ }
+
+ if compare {
+ log.Printf("Fetching data from %v...\n", toSrv.Name())
+ toTimer = time.Now()
+ if err := toSrv.Fetch(); err != nil {
+ log.Fatalf("Unable to fetch toSrv data: %v\n", err)
+ }
+
+ toSSLKeys, toDNSSecKeys, toURIKeys, toURLKeys, err := GetKeys(toSrv)
+ if err != nil {
+ log.Fatal(err)
+ }
+ log.Println("Validating " + toSrv.Name())
+ if err := toSrv.ValidateKey(); err != nil && len(err) > 0 {
+ log.Fatal(strings.Join(err, "\n"))
+ }
+ log.Printf("Done [%v seconds]\n", time.Now().Sub(toTimer).Seconds())
+ log.Println(fromSrv.String())
+ log.Println(toSrv.String())
+
+ if !reflect.DeepEqual(fromSSLKeys, toSSLKeys) {
+ log.Fatal("from sslkeys and to sslkeys don't match")
+ }
+ if !reflect.DeepEqual(fromDNSSecKeys, toDNSSecKeys) {
+ log.Fatal("from dnssec and to dnssec don't match")
+ }
+ if !reflect.DeepEqual(fromURIKeys, toURIKeys) {
+ log.Fatal("from uri and to uri don't match")
+ }
+ if !reflect.DeepEqual(fromURLKeys, toURLKeys) {
+ log.Fatal("from url and to url don't match")
+ }
+ log.Println("Both datasources have the same keys!")
+ return
+ }
+
+ log.Printf("Setting %v keys...\n", toSrv.Name())
+ toTimer = time.Now()
+ if err := SetKeys(toSrv, fromSSLKeys, fromDNSSecKeys, fromURIKeys, fromURLKeys); err != nil {
+ log.Fatal(err)
+ }
+
+ if err := Validate(toSrv); err != nil {
+ log.Fatal(err)
+ }
+ log.Printf("Done [%v seconds]\n", time.Now().Sub(toTimer).Seconds())
+
+ log.Println(fromSrv.String())
+
+ if dry {
+ return
+ }
+
+ if confirm {
+ ans := "q"
+ for {
+ fmt.Print("Confirm data insertion (y/n): ")
+ if _, err := fmt.Scanln(&ans); err != nil {
+ log.Fatal("unable to get user input")
+ }
+
+ if ans == "y" {
+ break
+ } else if ans == "n" {
+ return
+ }
+ }
+ }
+ log.Printf("Inserting data into %v...\n", toSrv.Name())
+ toTimer = time.Now()
+ if err := toSrv.Insert(); err != nil {
+ log.Fatal(err)
+ }
+ log.Printf("Done [%v seconds]\n", time.Now().Sub(toTimer).Seconds())
+
+}
+
+// Validate runs the ValidateKey method on the backend
+func Validate(be TVBackend) error {
+ if errs := be.ValidateKey(); errs != nil && len(errs) > 0 {
+ return errors.New(fmt.Sprintf("Validation Errors (%v): \n%v", be.Name(), strings.Join(errs, "\n")))
+ } else {
+ log.Println("Validated " + be.Name())
+ }
+ return nil
+}
+
+// SetKeys will set all of the keys for a backend
+func SetKeys(be TVBackend, sslkeys []SSLKey, dnssecKeys []DNSSecKey, uriKeys []URISignKey, urlKeys []URLSigKey) error {
+ if err := be.SetSSLKeys(sslkeys); err != nil {
+ return errors.New(fmt.Sprintf("Unable to set %v ssl keys: %v", be.Name(), err))
+ }
+ if err := be.SetDNSSecKeys(dnssecKeys); err != nil {
+ return errors.New(fmt.Sprintf("Unable to set %v dnssec keys: %v", be.Name(), err))
+ }
+ if err := be.SetURLSigKeys(urlKeys); err != nil {
+ return errors.New(fmt.Sprintf("Unable to set %v url keys: %v", be.Name(), err))
+ }
+ if err := be.SetURISignKeys(uriKeys); err != nil {
+ return errors.New(fmt.Sprintf("Unable to set %v uri keys: %v", be.Name(), err))
+ }
+ return nil
+}
+
+// GetKeys will get all of the keys for a backend
+func GetKeys(be TVBackend) ([]SSLKey, []DNSSecKey, []URISignKey, []URLSigKey, error) {
Review comment:
this function is a bit unwieldy with 5 outputs -- could we lump the keys altogether into a `type Secrets struct {SSLKeys, ...}`?
##########
File path: tools/traffic_vault_migrate/traffic_vault_migrate.go
##########
@@ -0,0 +1,448 @@
+package main
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "reflect"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+var (
+ fromType string
+ toType string
+ fromCfg string
+ toCfg string
+ dry bool
+ compare bool
+ confirm bool
+ dump bool
+
+ riakBE RiakBackend = RiakBackend{}
+ pgBE PGBackend = PGBackend{}
+)
+
+func init() {
+ flag.StringVar(&fromType, "from_type", riakBE.Name(), fmt.Sprintf("From server types (%v)", strings.Join(supportedTypes(), "|")))
+ flag.StringVar(&toType, "to_type", pgBE.Name(), fmt.Sprintf("From server types (%v)", strings.Join(supportedTypes(), "|")))
+ flag.StringVar(&toCfg, "to_cfg", "pg.json", "To server config file")
+ flag.StringVar(&fromCfg, "from_cfg", "riak.json", "From server config file")
+ flag.BoolVar(&dry, "dry", false, "Do not perform writes")
+ flag.BoolVar(&compare, "compare", false, "Compare to and from server records")
+ flag.BoolVar(&confirm, "confirm", true, "Requires confirmation before inserting records")
+ flag.BoolVar(&dump, "dump", false, "Write keys (from 'from' server) to disk")
+}
+
+func supportedBackends() []TVBackend {
+ return []TVBackend{
+ &riakBE, &pgBE,
+ }
+}
+
+func main() {
+ flag.Parse()
+ var fromSrv TVBackend
+ var toSrv TVBackend
+
+ toSrvUsed := !dump && !dry
+
+ if !validateType(fromType) {
+ log.Fatal("Unknown fromType " + fromType)
+ }
+ if toSrvUsed && !validateType(toType) {
+ log.Fatal("Unknown toType " + toType)
+ }
+
+ fromSrv = getBackendFromType(fromType)
+ if toSrvUsed {
+ toSrv = getBackendFromType(toType)
+ }
+
+ var toTimer time.Time
+ var toTime float64
+ var fromTimer time.Time
+ var fromTime float64
+
+ log.Println("Reading configs...")
+ fromTimer = time.Now()
+ if err := fromSrv.ReadConfig(fromCfg); err != nil {
+ log.Fatalf("Unable to read fromSrv cfg: %v", err)
+ }
+ fromTime = time.Now().Sub(fromTimer).Seconds()
+
+ if toSrvUsed {
+ toTimer = time.Now()
+ if err := toSrv.ReadConfig(toCfg); err != nil {
+ log.Fatalf("Unable to read toSrv cfg: %v", err)
+ }
+ toTime := time.Now().Sub(toTimer).Seconds()
+ log.Printf("Done [%v seconds]\n\tto: [%v seconds]\n\tfrom: [%v seconds]\n", toTime+fromTime, toTime, fromTime)
+ } else {
+ log.Printf("Done [%v seconds]\n", fromTime)
+ }
+
+ log.Println("Starting servers...")
Review comment:
Should this be "starting server connections..." instead?
##########
File path: tools/traffic_vault_migrate/traffic_vault_migrate.go
##########
@@ -0,0 +1,448 @@
+package main
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+ "encoding/json"
+ "errors"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "reflect"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/apache/trafficcontrol/lib/go-tc"
+)
+
+var (
+ fromType string
+ toType string
+ fromCfg string
+ toCfg string
+ dry bool
+ compare bool
+ confirm bool
+ dump bool
+
+ riakBE RiakBackend = RiakBackend{}
+ pgBE PGBackend = PGBackend{}
+)
+
+func init() {
+ flag.StringVar(&fromType, "from_type", riakBE.Name(), fmt.Sprintf("From server types (%v)", strings.Join(supportedTypes(), "|")))
+ flag.StringVar(&toType, "to_type", pgBE.Name(), fmt.Sprintf("From server types (%v)", strings.Join(supportedTypes(), "|")))
+ flag.StringVar(&toCfg, "to_cfg", "pg.json", "To server config file")
+ flag.StringVar(&fromCfg, "from_cfg", "riak.json", "From server config file")
+ flag.BoolVar(&dry, "dry", false, "Do not perform writes")
+ flag.BoolVar(&compare, "compare", false, "Compare to and from server records")
+ flag.BoolVar(&confirm, "confirm", true, "Requires confirmation before inserting records")
+ flag.BoolVar(&dump, "dump", false, "Write keys (from 'from' server) to disk")
+}
+
+func supportedBackends() []TVBackend {
+ return []TVBackend{
+ &riakBE, &pgBE,
+ }
+}
+
+func main() {
+ flag.Parse()
+ var fromSrv TVBackend
+ var toSrv TVBackend
+
+ toSrvUsed := !dump && !dry
+
+ if !validateType(fromType) {
+ log.Fatal("Unknown fromType " + fromType)
+ }
+ if toSrvUsed && !validateType(toType) {
+ log.Fatal("Unknown toType " + toType)
+ }
+
+ fromSrv = getBackendFromType(fromType)
+ if toSrvUsed {
+ toSrv = getBackendFromType(toType)
+ }
+
+ var toTimer time.Time
+ var toTime float64
+ var fromTimer time.Time
+ var fromTime float64
+
+ log.Println("Reading configs...")
+ fromTimer = time.Now()
+ if err := fromSrv.ReadConfig(fromCfg); err != nil {
+ log.Fatalf("Unable to read fromSrv cfg: %v", err)
+ }
+ fromTime = time.Now().Sub(fromTimer).Seconds()
+
+ if toSrvUsed {
+ toTimer = time.Now()
+ if err := toSrv.ReadConfig(toCfg); err != nil {
+ log.Fatalf("Unable to read toSrv cfg: %v", err)
+ }
+ toTime := time.Now().Sub(toTimer).Seconds()
Review comment:
All this timing code is a bit distracting -- is it really necessary to know how long each step takes? Also, if we really needed to know, we should be able to use the log timestamps from "starting ..." to the next "starting..." message, right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org