You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by GitBox <gi...@apache.org> on 2021/09/01 09:22:24 UTC
[GitHub] [apisix-dashboard] RedemptionC commented on a change in pull request #2065: feat: add cache verification tool for dashbaord
RedemptionC commented on a change in pull request #2065:
URL: https://github.com/apache/apisix-dashboard/pull/2065#discussion_r699496206
##########
File path: api/cmd/cache_verify.go
##########
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/spf13/cobra"
+ "github.com/tidwall/gjson"
+
+ "github.com/apisix/manager-api/internal/conf"
+ "github.com/apisix/manager-api/internal/handler/cache_verify"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+var port int
+var host string
+
+var username, password string
+
+type response struct {
+ Data cache_verify.OutputResult `json:"data"`
+}
+
+func newCacheVerifyCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "cache-verify",
+ Short: "verify that data in cache are consistent with that in ETCD",
+ Run: func(cmd *cobra.Command, args []string) {
+ conf.InitConf()
+ log.InitLogger()
+
+ port = conf.ServerPort
+ host = conf.ServerHost
+ username = "admin"
Review comment:
>@nic-chen
if we use account from the conf,should we use admin or user?
there are 2 user in the config file
##########
File path: api/cmd/cache_verify.go
##########
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/spf13/cobra"
+ "github.com/tidwall/gjson"
+
+ "github.com/apisix/manager-api/internal/conf"
+ "github.com/apisix/manager-api/internal/handler/cache_verify"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+var port int
+var host string
+
+var username, password string
+
+type response struct {
+ Data cache_verify.OutputResult `json:"data"`
+}
+
+func newCacheVerifyCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "cache-verify",
+ Short: "verify that data in cache are consistent with that in ETCD",
+ Run: func(cmd *cobra.Command, args []string) {
+ conf.InitConf()
+ log.InitLogger()
+
+ port = conf.ServerPort
+ host = conf.ServerHost
+ username = "admin"
+ password = conf.UserList[username].Password
+
+ token := getToken()
+
+ url := fmt.Sprintf("http://%s:%d/apisix/admin/cache_verify", host, port)
+ client := &http.Client{}
+
+ get, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Errorf("new http request failed: %s", err)
+ return
+ }
+
+ get.Header.Set("Authorization", token)
+
+ rsp, err := client.Do(get)
+ if err != nil {
+ log.Errorf("get result from migrate/export failed: %s", err)
+ return
+ }
+ defer func() {
+ err := rsp.Body.Close()
+ if err != nil {
+ log.Errorf("close on response body failed: %s", err)
+ return
+ }
+ }()
+
+ data, err := ioutil.ReadAll(rsp.Body)
+ if err != nil {
+ log.Errorf("io read all failed: %s", err)
+ return
+ }
+
+ var rs response
+ err = json.Unmarshal(data, &rs)
+ if err != nil {
+ log.Errorf("bad Data format,json unmarshal failed: %s", err)
+ return
+ }
+
+ fmt.Printf("cache verification result as follows:\n\n")
+ fmt.Printf("There are %d items in total,%d of them are consistent,%d of them are inconsistent\n",
+ rs.Data.Total, rs.Data.ConsistentCount, rs.Data.InconsistentCount)
+
+ names := []string{
+ "ssls",
+ "routes",
+ "scripts",
+ "services",
+ "upstreams",
+ "consumers",
+ "server infos",
+ "global plugins",
+ "plugin configs",
+ }
+ datas := []cache_verify.StatisticalData{
+ rs.Data.Items.SSLs,
+ rs.Data.Items.Routes,
+ rs.Data.Items.Scripts,
+ rs.Data.Items.Services,
+ rs.Data.Items.Upstreams,
+ rs.Data.Items.Consumers,
+ rs.Data.Items.ServerInfos,
+ rs.Data.Items.GlobalPlugins,
+ rs.Data.Items.PluginConfigs,
+ }
Review comment:
because map doesn't guarantee iteration order,I want to make sure the output are sorted by the config name,like:
```
❯ go run main.go cache-verify
cache verification result as follows:
There are 12 items in total,12 of them are consistent,0 of them are inconsistent
ssls : 0 in total,0 consistent,0 inconsistent
routes : 1 in total,1 consistent,0 inconsistent
scripts : 0 in total,0 consistent,0 inconsistent
services : 1 in total,1 consistent,0 inconsistent
upstreams : 2 in total,2 consistent,0 inconsistent
consumers : 7 in total,7 consistent,0 inconsistent
server infos : 1 in total,1 consistent,0 inconsistent
global plugins : 0 in total,0 consistent,0 inconsistent
plugin configs : 0 in total,0 consistent,0 inconsistent
```
##########
File path: api/internal/handler/cache_verify/cache_verify_test.go
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cache_verify
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+
+ "github.com/apisix/manager-api/internal/core/storage"
+ "github.com/apisix/manager-api/internal/core/store"
+ "github.com/apisix/manager-api/internal/log"
+ "github.com/shiningrush/droplet"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+)
+
+func TestHandler_CacheVerify(t *testing.T) {
+ andyStr := `{"username":"andy","plugins":{"key-auth":{"key":"key-of-john"}},"create_time":1627739045,"update_time":1627744978}`
+ var andyObj interface{}
+ err := json.Unmarshal([]byte(andyStr), &andyObj)
+ if err != nil {
+ log.Errorf("unmarshal error :: %s", err)
+ return
+ }
+ brokenAndyStr := `{"username":"andy","plugins":{"key-auth":{"key":"key-of-john"}},"create_time":1627739046,"update_time":1627744978}`
+ var brokenAndyObj interface{}
+ err = json.Unmarshal([]byte(brokenAndyStr), &brokenAndyObj)
+ if err != nil {
+ log.Errorf("unmarshal error :: %s", err)
+ return
+ }
+ consumerPrefix := "/apisix/consumers/"
+
+ tests := []struct {
+ caseDesc string
+ listInput string
+ listRet []storage.Keypair
+ getInput string
+ getRet interface{}
+ wantInconsistentConsumer int
+ }{
+ {
+ caseDesc: "consistent",
+ listInput: consumerPrefix,
+ listRet: []storage.Keypair{
+ {
+ Key: consumerPrefix + "andy",
+ Value: andyStr,
+ },
+ },
+ getInput: "andy",
+ getRet: andyObj,
+ wantInconsistentConsumer: 0,
+ },
+ {
+ caseDesc: "inconsistent",
+ listInput: consumerPrefix,
+ listRet: []storage.Keypair{
+ {
+ Key: consumerPrefix + "andy",
+ Value: andyStr,
+ },
+ },
+ getInput: "andy",
+ getRet: brokenAndyObj,
+ wantInconsistentConsumer: 1,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.caseDesc, func(t *testing.T) {
+ mockConsumerCache := store.MockInterface{}
+ mockEtcdStorage := storage.MockInterface{}
+ mockEtcdStorage.On("List", context.TODO(), consumerPrefix).Return(tc.listRet, nil)
+ // for any other type of configs,etcd.List just return empty slice,so it will not check further
+ mockEtcdStorage.On("List", context.TODO(), mock.Anything).Return([]storage.Keypair{}, nil)
+
+ mockConsumerCache.On("Get", tc.getInput).Return(tc.getRet, nil)
+ handler := Handler{consumerStore: &mockConsumerCache, etcdStorage: &mockEtcdStorage}
+ rs, err := handler.CacheVerify(droplet.NewContext())
+ assert.Nil(t, err, nil)
+ v, ok := rs.(OutputResult)
+ assert.True(t, ok, true)
+ assert.Equal(t, v.Items.Consumers.InconsistentCount, tc.wantInconsistentConsumer)
+
+ })
+ }
+
+}
Review comment:
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org