You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by GitBox <gi...@apache.org> on 2021/05/31 15:56:00 UTC
[GitHub] [apisix-dashboard] starsz commented on a change in pull request #1893: feat: add api of config migrate, export and import
starsz commented on a change in pull request #1893:
URL: https://github.com/apache/apisix-dashboard/pull/1893#discussion_r642574726
##########
File path: api/internal/handler/config_migrate/migrate.go
##########
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package migrate
+
+import (
+ "encoding/binary"
+ "io/ioutil"
+ "net/http"
+ "strconv"
+
+ "github.com/apisix/manager-api/internal/handler"
+ "github.com/apisix/manager-api/internal/utils/consts"
+ "github.com/gin-gonic/gin"
+ "github.com/shiningrush/droplet/data"
+
+ "hash/crc32"
+
+ "github.com/apisix/manager-api/internal/core/migrate"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+const (
+ exportFileName = "apisix-config.bak"
+)
+
+type Handler struct{}
+
+func NewHandler() (handler.RouteRegister, error) {
+ return &Handler{}, nil
+}
+
+func (h *Handler) ApplyRoute(r *gin.Engine) {
+ r.GET("/apisix/admin/migrate/export", h.ExportConfig)
+ r.POST("/apisix/admin/migrate/import", h.ImportConfig)
+}
+
+type ExportInput struct{}
+
+func (h *Handler) ExportConfig(c *gin.Context) {
+ data, err := migrate.Export(c)
+ if err != nil {
+ log.Errorf("Export: %s", err)
+ c.JSON(http.StatusInternalServerError, err)
+ return
+ }
+ // To check file integrity
+ // Add 4 byte(uint32) checksum at the end of file.
+ checksumUint32 := crc32.ChecksumIEEE(data)
+ checksum := make([]byte, 4)
+ binary.BigEndian.PutUint32(checksum, checksumUint32)
+ fileBytes := append(data, checksum...)
+
+ c.Writer.WriteHeader(http.StatusOK)
+ c.Header("Content-Disposition", "attachment; filename="+exportFileName)
+ c.Header("Content-Type", "application/octet-stream")
+ c.Header("Accept-Length", strconv.Itoa(len(fileBytes)))
+ c.Header("Content-Transfer-Encoding", "binary")
+ _, err = c.Writer.Write([]byte(fileBytes))
+ if err != nil {
+ log.Errorf("Write: %s", err)
+ }
Review comment:
Need to give an error response.
##########
File path: api/test/e2enew/migrate/migrate_test.go
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrate
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/json"
+ "fmt"
+ "hash/crc32"
+ "net/http"
+
+ "github.com/onsi/ginkgo"
+
+ "github.com/apisix/manager-api/test/e2enew/base"
+)
+
+type response struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+var _ = ginkgo.Describe("Migrate", func() {
+ var exportData []byte
+
+ ginkgo.It("prepare config data", prepareConfigData)
+ ginkgo.It("export config success", func() {
+ req := base.ManagerApiExpect().GET("/apisix/admin/migrate/export")
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ exportData = []byte(resp.Body().Raw())
+ data := exportData[:len(exportData)-4]
Review comment:
Just want to know why we should "-4"
##########
File path: api/internal/core/migrate/migrate.go
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrate
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/apisix/manager-api/internal/core/store"
+)
+
+var (
+ ErrConflict = errors.New("conflict")
+)
+
+func Export(ctx context.Context) ([]byte, error) {
+ exportData := NewAllData()
+ store.RangeStore(func(key store.HubKey, s *store.GenericStore) bool {
+ s.Range(ctx, func(_ string, obj interface{}) bool {
+ err := exportData.AddObj(obj)
+ if err != nil {
Review comment:
It's better to add some log here.
##########
File path: api/internal/handler/config_migrate/migrate.go
##########
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package migrate
+
+import (
+ "encoding/binary"
+ "io/ioutil"
+ "net/http"
+ "strconv"
+
+ "github.com/apisix/manager-api/internal/handler"
+ "github.com/apisix/manager-api/internal/utils/consts"
+ "github.com/gin-gonic/gin"
+ "github.com/shiningrush/droplet/data"
+
+ "hash/crc32"
+
+ "github.com/apisix/manager-api/internal/core/migrate"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+const (
+ exportFileName = "apisix-config.bak"
+)
+
+type Handler struct{}
+
+func NewHandler() (handler.RouteRegister, error) {
+ return &Handler{}, nil
+}
+
+func (h *Handler) ApplyRoute(r *gin.Engine) {
+ r.GET("/apisix/admin/migrate/export", h.ExportConfig)
+ r.POST("/apisix/admin/migrate/import", h.ImportConfig)
+}
+
+type ExportInput struct{}
+
+func (h *Handler) ExportConfig(c *gin.Context) {
+ data, err := migrate.Export(c)
+ if err != nil {
+ log.Errorf("Export: %s", err)
+ c.JSON(http.StatusInternalServerError, err)
+ return
+ }
+ // To check file integrity
+ // Add 4 byte(uint32) checksum at the end of file.
+ checksumUint32 := crc32.ChecksumIEEE(data)
+ checksum := make([]byte, 4)
+ binary.BigEndian.PutUint32(checksum, checksumUint32)
+ fileBytes := append(data, checksum...)
+
+ c.Writer.WriteHeader(http.StatusOK)
+ c.Header("Content-Disposition", "attachment; filename="+exportFileName)
+ c.Header("Content-Type", "application/octet-stream")
+ c.Header("Accept-Length", strconv.Itoa(len(fileBytes)))
+ c.Header("Content-Transfer-Encoding", "binary")
+ _, err = c.Writer.Write([]byte(fileBytes))
+ if err != nil {
+ log.Errorf("Write: %s", err)
+ }
+}
+
+type ImportOutput struct {
+ ConflictItems *migrate.AllData
+}
+
+var modeMap = map[string]migrate.ConflictMode{
+ "return": migrate.ModeReturn,
+ "overwrite": migrate.ModeOverwrite,
+ "skip": migrate.ModeSkip,
+}
+
+func (h *Handler) ImportConfig(c *gin.Context) {
+ paraMode := c.PostForm("mode")
+ mode := migrate.ModeReturn
+ if m, ok := modeMap[paraMode]; ok {
+ mode = m
+ }
+ file, _, err := c.Request.FormFile("file")
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, err)
+ return
+ }
+ content, err := ioutil.ReadAll(file)
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, err)
+ return
+ }
+ // checksum uint32,4 bytes
+ importData := content[:len(content)-4]
+ checksum := binary.BigEndian.Uint32(content[len(content)-4:])
+ if checksum != crc32.ChecksumIEEE(importData) {
+ c.JSON(http.StatusOK, &data.BaseError{
+ Code: consts.ErrBadRequest,
+ Message: "Checksum check failure,maybe file broken",
+ })
+ return
+ }
+ conflictData, err := migrate.Import(c, importData, mode)
+ if err != nil {
+ log.Errorf("Import: %s", err)
Review comment:
The log is too simple.
##########
File path: api/test/e2enew/migrate/migrate_test.go
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrate
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/json"
+ "fmt"
+ "hash/crc32"
+ "net/http"
+
+ "github.com/onsi/ginkgo"
+
+ "github.com/apisix/manager-api/test/e2enew/base"
+)
+
+type response struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+var _ = ginkgo.Describe("Migrate", func() {
+ var exportData []byte
+
+ ginkgo.It("prepare config data", prepareConfigData)
+ ginkgo.It("export config success", func() {
+ req := base.ManagerApiExpect().GET("/apisix/admin/migrate/export")
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ exportData = []byte(resp.Body().Raw())
+ data := exportData[:len(exportData)-4]
+ checksum := binary.BigEndian.Uint32(exportData[len(exportData)-4:])
+ if checksum != crc32.ChecksumIEEE(data) {
+ ginkgo.Fail("Checksum not correct")
+ }
+ })
+
+ ginkgo.It("import config conflict and return", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "return"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 20001 {
+ ginkgo.Fail("code not 20001")
+ }
+ })
+
+ ginkgo.It("import config conflict and skip", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "skip"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("import config conflict and overwrite", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "overwrite"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("delete all config", deleteConfigData)
+
+ ginkgo.It("delete imported route failed", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/routes/r1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusNotFound,
+ })
+ })
+
+ ginkgo.It("import config success", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "return"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("delete imported route success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/routes/r1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+ ginkgo.It("delete imported upstream success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/upstreams/1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+ ginkgo.It("delete imported service success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/services/s1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+})
+
+func prepareConfigData() {
+ headers := map[string]string{
+ "Content-Type": "application/json",
+ "Authorization": base.GetToken(),
+ }
+ _, statusCode, err := base.HttpPut(base.ManagerAPIHost+"/apisix/admin/routes/r1", headers, `{
+ "name": "route1",
+ "uri": "/hello_",
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ }
+ }`)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
+ }
+ _, statusCode, err = base.HttpPut(base.ManagerAPIHost+"/apisix/admin/upstreams/1", headers, `{"name":"upstream1","nodes":[{"host":"172.16.238.20","port":1980,"weight":1}],"type":"roundrobin"}`)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
+ }
+ _, statusCode, err = base.HttpPut(base.ManagerAPIHost+"/apisix/admin/services/s1", headers, `{"name":"testservice","upstream":{"nodes":[{"host":"172.16.238.20","port":1980,"weight":1},{"host":"172.16.238.20","port":1981,"weight":2},{"host":"172.16.238.20","port":1982,"weight":3}],"type":"roundrobin"}}`)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
+ }
+}
+
+func deleteConfigData() {
+ headers := map[string]string{
+ "Content-Type": "application/json",
+ "Authorization": base.GetToken(),
+ }
+ _, statusCode, err := base.HttpDelete(base.ManagerAPIHost+"/apisix/admin/routes/r1", headers)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
+ }
+ _, statusCode, err = base.HttpDelete(base.ManagerAPIHost+"/apisix/admin/upstreams/1", headers)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
+ }
+ _, statusCode, err = base.HttpDelete(base.ManagerAPIHost+"/apisix/admin/services/s1", headers)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
+ }
+}
+
+const jsonConfig = `{"Counsumers":[],"Routes":[{"id":"r1","create_time":1620922236,"update_time":1620922236,"uri":"/hello_","name":"route1","upstream":{"nodes":{"127.0.0.1:1980":1},"type":"roundrobin"},"status":1}],"Services":[{"id":"s1","create_time":1620922236,"update_time":1620922236,"name":"testservice","upstream":{"nodes":[{"host":"172.16.238.20","port":1980,"weight":1},{"host":"172.16.238.20","port":1981,"weight":2},{"host":"172.16.238.20","port":1982,"weight":3}],"type":"roundrobin"}}],"SSLs":[],"Upstreams":[{"id":"1","create_time":1620922236,"update_time":1620922236,"nodes":[{"host":"172.16.238.20","port":1980,"weight":1}],"type":"roundrobin","name":"upstream1"}],"Scripts":[],"GlobalPlugins":[],"PluginConfigs":[]}`
Review comment:
The style looks not very good.
##########
File path: api/internal/core/migrate/conflict.go
##########
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrate
+
+import (
+ "context"
+
+ "github.com/apisix/manager-api/internal/core/store"
+)
+
+func isConflict(ctx context.Context, new *AllData) (bool, *AllData) {
+ isConflict := false
+ conflict := NewAllData()
+ store.RangeStore(func(key store.HubKey, s *store.GenericStore) bool {
+ new.Range(key, func(i int, obj interface{}) bool {
+ // Only check key of store conflict for now.
+ // TODO: Maybe check name of some entiries.
+ _, err := s.CreateCheck(obj)
+ if err != nil {
+ isConflict = true
+ err = conflict.AddObj(obj)
+ if err != nil {
Review comment:
Ditto.
##########
File path: api/internal/core/migrate/migrate.go
##########
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrate
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/apisix/manager-api/internal/core/store"
+)
+
+var (
+ ErrConflict = errors.New("conflict")
+)
+
+func Export(ctx context.Context) ([]byte, error) {
+ exportData := NewAllData()
+ store.RangeStore(func(key store.HubKey, s *store.GenericStore) bool {
+ s.Range(ctx, func(_ string, obj interface{}) bool {
+ err := exportData.AddObj(obj)
+ if err != nil {
+ return true
+ }
+ return true
+ })
+ return true
+ })
+
+ data, err := json.Marshal(exportData)
+ if err != nil {
+ return nil, err
+ }
+
+ return data, nil
+}
+
+type ConflictMode int
+
+const (
+ ModeReturn ConflictMode = iota
+ ModeOverwrite
+ ModeSkip
+)
+
+func Import(ctx context.Context, data []byte, mode ConflictMode) (*AllData, error) {
+ importData := NewAllData()
+ e := json.Unmarshal(data, &importData)
+ if e != nil {
+ return nil, e
+ }
+ conflict, conflictData := isConflict(ctx, importData)
+ if conflict && mode == ModeReturn {
+ return conflictData, ErrConflict
+ }
+ var err error
+ store.RangeStore(func(key store.HubKey, s *store.GenericStore) bool {
+ importData.Range(key, func(i int, obj interface{}) bool {
+ _, e := s.CreateCheck(obj)
+ if e != nil {
+ switch mode {
+ case ModeSkip:
+ return true
+ case ModeOverwrite:
+ _, e := s.Update(ctx, obj, true)
+ if e != nil {
+ err = e
+ return false
+ }
+ }
+ } else {
+ _, e := s.Create(ctx, obj)
+ if err != nil {
+ err = e
+ return false
+ }
+ }
+ fmt.Printf("[%s]: %#v\n", key, obj)
Review comment:
Please delete the debug info.
##########
File path: api/internal/handler/config_migrate/migrate.go
##########
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package migrate
+
+import (
+ "encoding/binary"
+ "io/ioutil"
+ "net/http"
+ "strconv"
+
+ "github.com/apisix/manager-api/internal/handler"
+ "github.com/apisix/manager-api/internal/utils/consts"
+ "github.com/gin-gonic/gin"
+ "github.com/shiningrush/droplet/data"
+
+ "hash/crc32"
Review comment:
You can move it in the front.
##########
File path: api/internal/handler/config_migrate/migrate.go
##########
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package migrate
+
+import (
+ "encoding/binary"
+ "io/ioutil"
+ "net/http"
+ "strconv"
+
+ "github.com/apisix/manager-api/internal/handler"
+ "github.com/apisix/manager-api/internal/utils/consts"
+ "github.com/gin-gonic/gin"
+ "github.com/shiningrush/droplet/data"
+
+ "hash/crc32"
+
+ "github.com/apisix/manager-api/internal/core/migrate"
+ "github.com/apisix/manager-api/internal/log"
+)
+
+const (
+ exportFileName = "apisix-config.bak"
+)
+
+type Handler struct{}
+
+func NewHandler() (handler.RouteRegister, error) {
+ return &Handler{}, nil
+}
+
+func (h *Handler) ApplyRoute(r *gin.Engine) {
+ r.GET("/apisix/admin/migrate/export", h.ExportConfig)
+ r.POST("/apisix/admin/migrate/import", h.ImportConfig)
+}
+
+type ExportInput struct{}
+
+func (h *Handler) ExportConfig(c *gin.Context) {
+ data, err := migrate.Export(c)
+ if err != nil {
+ log.Errorf("Export: %s", err)
+ c.JSON(http.StatusInternalServerError, err)
+ return
+ }
+ // To check file integrity
+ // Add 4 byte(uint32) checksum at the end of file.
+ checksumUint32 := crc32.ChecksumIEEE(data)
+ checksum := make([]byte, 4)
+ binary.BigEndian.PutUint32(checksum, checksumUint32)
+ fileBytes := append(data, checksum...)
+
+ c.Writer.WriteHeader(http.StatusOK)
+ c.Header("Content-Disposition", "attachment; filename="+exportFileName)
+ c.Header("Content-Type", "application/octet-stream")
+ c.Header("Accept-Length", strconv.Itoa(len(fileBytes)))
Review comment:
What's the meaning of this head?
##########
File path: api/test/e2enew/migrate/migrate_test.go
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrate
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/json"
+ "fmt"
+ "hash/crc32"
+ "net/http"
+
+ "github.com/onsi/ginkgo"
+
+ "github.com/apisix/manager-api/test/e2enew/base"
+)
+
+type response struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+var _ = ginkgo.Describe("Migrate", func() {
+ var exportData []byte
+
+ ginkgo.It("prepare config data", prepareConfigData)
+ ginkgo.It("export config success", func() {
+ req := base.ManagerApiExpect().GET("/apisix/admin/migrate/export")
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ exportData = []byte(resp.Body().Raw())
+ data := exportData[:len(exportData)-4]
+ checksum := binary.BigEndian.Uint32(exportData[len(exportData)-4:])
+ if checksum != crc32.ChecksumIEEE(data) {
+ ginkgo.Fail("Checksum not correct")
+ }
+ })
+
+ ginkgo.It("import config conflict and return", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "return"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 20001 {
+ ginkgo.Fail("code not 20001")
+ }
+ })
+
+ ginkgo.It("import config conflict and skip", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "skip"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("import config conflict and overwrite", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "overwrite"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("delete all config", deleteConfigData)
+
+ ginkgo.It("delete imported route failed", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/routes/r1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusNotFound,
+ })
+ })
+
+ ginkgo.It("import config success", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "return"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("delete imported route success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/routes/r1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+ ginkgo.It("delete imported upstream success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/upstreams/1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+ ginkgo.It("delete imported service success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/services/s1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+})
+
+func prepareConfigData() {
+ headers := map[string]string{
+ "Content-Type": "application/json",
+ "Authorization": base.GetToken(),
+ }
+ _, statusCode, err := base.HttpPut(base.ManagerAPIHost+"/apisix/admin/routes/r1", headers, `{
+ "name": "route1",
+ "uri": "/hello_",
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ }
+ }`)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
+ }
+ _, statusCode, err = base.HttpPut(base.ManagerAPIHost+"/apisix/admin/upstreams/1", headers, `{"name":"upstream1","nodes":[{"host":"172.16.238.20","port":1980,"weight":1}],"type":"roundrobin"}`)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
+ }
+ _, statusCode, err = base.HttpPut(base.ManagerAPIHost+"/apisix/admin/services/s1", headers, `{"name":"testservice","upstream":{"nodes":[{"host":"172.16.238.20","port":1980,"weight":1},{"host":"172.16.238.20","port":1981,"weight":2},{"host":"172.16.238.20","port":1982,"weight":3}],"type":"roundrobin"}}`)
+ if statusCode != http.StatusOK || err != nil {
Review comment:
The style looks not very good.
##########
File path: api/test/e2enew/migrate/migrate_test.go
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrate
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/json"
+ "fmt"
+ "hash/crc32"
+ "net/http"
+
+ "github.com/onsi/ginkgo"
+
+ "github.com/apisix/manager-api/test/e2enew/base"
+)
+
+type response struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+var _ = ginkgo.Describe("Migrate", func() {
+ var exportData []byte
+
+ ginkgo.It("prepare config data", prepareConfigData)
+ ginkgo.It("export config success", func() {
+ req := base.ManagerApiExpect().GET("/apisix/admin/migrate/export")
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ exportData = []byte(resp.Body().Raw())
+ data := exportData[:len(exportData)-4]
+ checksum := binary.BigEndian.Uint32(exportData[len(exportData)-4:])
+ if checksum != crc32.ChecksumIEEE(data) {
+ ginkgo.Fail("Checksum not correct")
+ }
+ })
+
+ ginkgo.It("import config conflict and return", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "return"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 20001 {
+ ginkgo.Fail("code not 20001")
+ }
+ })
+
+ ginkgo.It("import config conflict and skip", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "skip"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("import config conflict and overwrite", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "overwrite"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("delete all config", deleteConfigData)
+
+ ginkgo.It("delete imported route failed", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/routes/r1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusNotFound,
+ })
+ })
+
+ ginkgo.It("import config success", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "return"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
+ if err != nil {
+ ginkgo.Fail("json unmarshal:" + err.Error())
+ }
+ if rsp.Code != 0 {
+ ginkgo.Fail("code not 0")
+ }
+ })
+
+ ginkgo.It("delete imported route success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/routes/r1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+ ginkgo.It("delete imported upstream success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/upstreams/1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+ ginkgo.It("delete imported service success", func() {
+ base.RunTestCase(base.HttpTestCase{
+ Object: base.ManagerApiExpect(),
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/services/s1",
+ Headers: map[string]string{"Authorization": base.GetToken()},
+ ExpectStatus: http.StatusOK,
+ })
+ })
+
+})
+
+func prepareConfigData() {
+ headers := map[string]string{
+ "Content-Type": "application/json",
+ "Authorization": base.GetToken(),
+ }
+ _, statusCode, err := base.HttpPut(base.ManagerAPIHost+"/apisix/admin/routes/r1", headers, `{
+ "name": "route1",
+ "uri": "/hello_",
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ }
+ }`)
+ if statusCode != http.StatusOK || err != nil {
+ panic(fmt.Sprintf("%d, %s", statusCode, err))
Review comment:
Hi, we use use `gomega` or assert instead.
##########
File path: api/test/e2enew/migrate/migrate_test.go
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package migrate
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/json"
+ "fmt"
+ "hash/crc32"
+ "net/http"
+
+ "github.com/onsi/ginkgo"
+
+ "github.com/apisix/manager-api/test/e2enew/base"
+)
+
+type response struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+var _ = ginkgo.Describe("Migrate", func() {
+ var exportData []byte
+
+ ginkgo.It("prepare config data", prepareConfigData)
+ ginkgo.It("export config success", func() {
+ req := base.ManagerApiExpect().GET("/apisix/admin/migrate/export")
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ exportData = []byte(resp.Body().Raw())
+ data := exportData[:len(exportData)-4]
+ checksum := binary.BigEndian.Uint32(exportData[len(exportData)-4:])
+ if checksum != crc32.ChecksumIEEE(data) {
+ ginkgo.Fail("Checksum not correct")
+ }
+ })
+
+ ginkgo.It("import config conflict and return", func() {
+ req := base.ManagerApiExpect().POST("/apisix/admin/migrate/import")
+ buffer := bytes.NewBuffer(exportData)
+ req.WithMultipart().WithForm(map[string]string{"mode": "return"})
+ req.WithMultipart().WithFile("file", "apisix-config.bak", buffer)
+ resp := req.Expect()
+ resp.Status(http.StatusOK)
+ rsp := &response{}
+ err := json.Unmarshal([]byte(resp.Body().Raw()), rsp)
Review comment:
Need to check the `rsp` content.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org