You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by GitBox <gi...@apache.org> on 2021/09/01 09:23:25 UTC
[GitHub] [apisix-dashboard] nic-chen commented on a change in pull request #2065: feat: add cache verification tool for dashbaord
nic-chen commented on a change in pull request #2065:
URL: https://github.com/apache/apisix-dashboard/pull/2065#discussion_r699467417
##########
File path: api/cmd/cache_verify.go
##########
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/spf13/cobra"
+ "github.com/tidwall/gjson"
+
+ "github.com/apisix/manager-api/internal/conf"
+ "github.com/apisix/manager-api/internal/handler/cache_verify"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+var port int
+var host string
+
+var username, password string
+
+type response struct {
+ Data cache_verify.OutputResult `json:"data"`
+}
+
+func newCacheVerifyCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "cache-verify",
+ Short: "verify that data in cache are consistent with that in ETCD",
+ Run: func(cmd *cobra.Command, args []string) {
+ conf.InitConf()
+ log.InitLogger()
+
+ port = conf.ServerPort
+ host = conf.ServerHost
+ username = "admin"
+ password = conf.UserList[username].Password
+
+ token := getToken()
+
+ url := fmt.Sprintf("http://%s:%d/apisix/admin/cache_verify", host, port)
+ client := &http.Client{}
+
+ get, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Errorf("new http request failed: %s", err)
+ return
+ }
+
+ get.Header.Set("Authorization", token)
+
+ rsp, err := client.Do(get)
+ if err != nil {
+ log.Errorf("get result from migrate/export failed: %s", err)
Review comment:
ditto
##########
File path: api/cmd/cache_verify.go
##########
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/spf13/cobra"
+ "github.com/tidwall/gjson"
+
+ "github.com/apisix/manager-api/internal/conf"
+ "github.com/apisix/manager-api/internal/handler/cache_verify"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+var port int
+var host string
+
+var username, password string
+
+type response struct {
+ Data cache_verify.OutputResult `json:"data"`
+}
+
+func newCacheVerifyCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "cache-verify",
+ Short: "verify that data in cache are consistent with that in ETCD",
+ Run: func(cmd *cobra.Command, args []string) {
+ conf.InitConf()
+ log.InitLogger()
+
+ port = conf.ServerPort
+ host = conf.ServerHost
+ username = "admin"
+ password = conf.UserList[username].Password
+
+ token := getToken()
+
+ url := fmt.Sprintf("http://%s:%d/apisix/admin/cache_verify", host, port)
+ client := &http.Client{}
+
+ get, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Errorf("new http request failed: %s", err)
Review comment:
```suggestion
fmt.Fprintf(os.Stderr, "new http request failed: %s\n", err)
```
##########
File path: api/cmd/cache_verify.go
##########
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/spf13/cobra"
+ "github.com/tidwall/gjson"
+
+ "github.com/apisix/manager-api/internal/conf"
+ "github.com/apisix/manager-api/internal/handler/cache_verify"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+var port int
+var host string
+
+var username, password string
Review comment:
```suggestion
var (
port int
host, username, password string
)
```
##########
File path: api/internal/handler/cache_verify/cache_verify_test.go
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cache_verify
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+
+ "github.com/apisix/manager-api/internal/core/storage"
+ "github.com/apisix/manager-api/internal/core/store"
+ "github.com/apisix/manager-api/internal/log"
+ "github.com/shiningrush/droplet"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+)
+
+func TestHandler_CacheVerify(t *testing.T) {
+ andyStr := `{"username":"andy","plugins":{"key-auth":{"key":"key-of-john"}},"create_time":1627739045,"update_time":1627744978}`
+ var andyObj interface{}
+ err := json.Unmarshal([]byte(andyStr), &andyObj)
+ if err != nil {
+ log.Errorf("unmarshal error :: %s", err)
+ return
+ }
+ brokenAndyStr := `{"username":"andy","plugins":{"key-auth":{"key":"key-of-john"}},"create_time":1627739046,"update_time":1627744978}`
+ var brokenAndyObj interface{}
+ err = json.Unmarshal([]byte(brokenAndyStr), &brokenAndyObj)
+ if err != nil {
+ log.Errorf("unmarshal error :: %s", err)
+ return
+ }
+ consumerPrefix := "/apisix/consumers/"
+
+ tests := []struct {
+ caseDesc string
+ listInput string
+ listRet []storage.Keypair
+ getInput string
+ getRet interface{}
+ wantInconsistentConsumer int
+ }{
+ {
+ caseDesc: "consistent",
+ listInput: consumerPrefix,
+ listRet: []storage.Keypair{
+ {
+ Key: consumerPrefix + "andy",
+ Value: andyStr,
+ },
+ },
+ getInput: "andy",
+ getRet: andyObj,
+ wantInconsistentConsumer: 0,
+ },
+ {
+ caseDesc: "inconsistent",
+ listInput: consumerPrefix,
+ listRet: []storage.Keypair{
+ {
+ Key: consumerPrefix + "andy",
+ Value: andyStr,
+ },
+ },
+ getInput: "andy",
+ getRet: brokenAndyObj,
+ wantInconsistentConsumer: 1,
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.caseDesc, func(t *testing.T) {
+ mockConsumerCache := store.MockInterface{}
+ mockEtcdStorage := storage.MockInterface{}
+ mockEtcdStorage.On("List", context.TODO(), consumerPrefix).Return(tc.listRet, nil)
+ // for any other type of configs,etcd.List just return empty slice,so it will not check further
+ mockEtcdStorage.On("List", context.TODO(), mock.Anything).Return([]storage.Keypair{}, nil)
+
+ mockConsumerCache.On("Get", tc.getInput).Return(tc.getRet, nil)
+ handler := Handler{consumerStore: &mockConsumerCache, etcdStorage: &mockEtcdStorage}
+ rs, err := handler.CacheVerify(droplet.NewContext())
+ assert.Nil(t, err, nil)
+ v, ok := rs.(OutputResult)
+ assert.True(t, ok, true)
+ assert.Equal(t, v.Items.Consumers.InconsistentCount, tc.wantInconsistentConsumer)
+
+ })
+ }
+
+}
Review comment:
What is the test coverage?
##########
File path: api/cmd/cache_verify.go
##########
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/spf13/cobra"
+ "github.com/tidwall/gjson"
+
+ "github.com/apisix/manager-api/internal/conf"
+ "github.com/apisix/manager-api/internal/handler/cache_verify"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+var port int
+var host string
+
+var username, password string
+
+type response struct {
+ Data cache_verify.OutputResult `json:"data"`
+}
+
+func newCacheVerifyCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "cache-verify",
+ Short: "verify that data in cache are consistent with that in ETCD",
+ Run: func(cmd *cobra.Command, args []string) {
+ conf.InitConf()
+ log.InitLogger()
+
+ port = conf.ServerPort
+ host = conf.ServerHost
+ username = "admin"
Review comment:
why not use username in `config.yaml`? it can be not `admin`.
##########
File path: api/cmd/cache_verify.go
##########
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/spf13/cobra"
+ "github.com/tidwall/gjson"
+
+ "github.com/apisix/manager-api/internal/conf"
+ "github.com/apisix/manager-api/internal/handler/cache_verify"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+var port int
+var host string
+
+var username, password string
+
+type response struct {
+ Data cache_verify.OutputResult `json:"data"`
+}
+
+func newCacheVerifyCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "cache-verify",
+ Short: "verify that data in cache are consistent with that in ETCD",
+ Run: func(cmd *cobra.Command, args []string) {
+ conf.InitConf()
+ log.InitLogger()
+
+ port = conf.ServerPort
+ host = conf.ServerHost
+ username = "admin"
+ password = conf.UserList[username].Password
+
+ token := getToken()
+
+ url := fmt.Sprintf("http://%s:%d/apisix/admin/cache_verify", host, port)
+ client := &http.Client{}
+
+ get, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ log.Errorf("new http request failed: %s", err)
+ return
+ }
+
+ get.Header.Set("Authorization", token)
+
+ rsp, err := client.Do(get)
+ if err != nil {
+ log.Errorf("get result from migrate/export failed: %s", err)
+ return
+ }
+ defer func() {
+ err := rsp.Body.Close()
+ if err != nil {
+ log.Errorf("close on response body failed: %s", err)
+ return
+ }
+ }()
+
+ data, err := ioutil.ReadAll(rsp.Body)
+ if err != nil {
+ log.Errorf("io read all failed: %s", err)
+ return
+ }
+
+ var rs response
+ err = json.Unmarshal(data, &rs)
+ if err != nil {
+ log.Errorf("bad Data format,json unmarshal failed: %s", err)
+ return
+ }
+
+ fmt.Printf("cache verification result as follows:\n\n")
+ fmt.Printf("There are %d items in total,%d of them are consistent,%d of them are inconsistent\n",
+ rs.Data.Total, rs.Data.ConsistentCount, rs.Data.InconsistentCount)
+
+ names := []string{
+ "ssls",
+ "routes",
+ "scripts",
+ "services",
+ "upstreams",
+ "consumers",
+ "server infos",
+ "global plugins",
+ "plugin configs",
+ }
+ datas := []cache_verify.StatisticalData{
+ rs.Data.Items.SSLs,
+ rs.Data.Items.Routes,
+ rs.Data.Items.Scripts,
+ rs.Data.Items.Services,
+ rs.Data.Items.Upstreams,
+ rs.Data.Items.Consumers,
+ rs.Data.Items.ServerInfos,
+ rs.Data.Items.GlobalPlugins,
+ rs.Data.Items.PluginConfigs,
+ }
Review comment:
why not use a map? and we could put them into the function `printResult`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org