[GitHub] [pulsar] flowchartsman edited a comment on issue #9200: Swagger is broken.

[GitBox <gi...@apache.org>]

flowchartsman edited a comment on issue #9200:
URL: https://github.com/apache/pulsar/issues/9200#issuecomment-759253839


   Attached is the code (in Go) I've attempted to use to resolve the duplicated operationIds through heuristics. It seems to work rather well, but I am unsure how to resolve the multiple body parameters:
   
   ```go
   package main
   
   import (
   	"bytes"
   	"encoding/json"
   	"flag"
   	"fmt"
   	"io"
   	"io/ioutil"
   	"log"
   	"net/http"
   	"os"
   	"path/filepath"
   	"strings"
   
   	"github.com/fatih/camelcase"
   	"github.com/go-openapi/analysis"
   	"github.com/go-openapi/loads"
   	"github.com/go-openapi/spec"
   )
   
   const pulsarVersion = `2.7.0`
   
   var specFiles = []string{
   	"swagger.json",
   	"swaggerfunctions.json",
   	"swaggersource.json",
   	"swaggersink.json",
   }
   
   type opConflict struct {
   	op     *spec.Operation
   	route  string
   	method string
   }
   
   type conflictList struct {
   	conflicts []*opConflict
   	unchanged []*opConflict
   }
   
   type opConflicts map[string]*conflictList
   
   func main() {
   	log.SetFlags(0)
   	specFile := flag.String("specfile", "", "an individual spec file to download and fix (no merge")
   	flag.Parse()
   	var (
   		specs []*spec.Swagger
   		err   error
   	)
   	if *specFile == "" {
   		specs, err = downloadSpecs(specFiles)
   		if err != nil {
   			log.Fatal(err)
   		}
   		for _, spec := range specs {
   			if err := resolveDuplicatOpIDs(spec); err != nil {
   				log.Fatalf("error resolving duplicate OpIds: %v", err)
   			}
   			if err := fixPaths(spec); err != nil {
   				log.Fatalf("error performing fixPaths on %s", spec.ID)
   			}
   		}
   		if err := ensureNoDuplicateDefinitions(specs); err != nil {
   			log.Fatalf("error resolving duplicate definitions: %v", err)
   		}
   		collisions := analysis.Mixin(specs[0], specs[1:]...)
   		for _, collision := range collisions {
   			log.Println(collision)
   		}
   	} else {
   		// otherwise download and fix a single spec file
   		specs, err = downloadSpecs([]string{*specFile})
   		if err != nil {
   			log.Fatal(err)
   		}
   		if err := resolveDuplicatOpIDs(specs[0]); err != nil {
   			log.Fatalf("error resolving duplicate OpIds: %v", err)
   		}
   	}
   	enc := json.NewEncoder(os.Stdout)
   	enc.SetIndent("", "  ")
   	enc.Encode(specs[0])
   }
   
   // download all specs
   func downloadSpecs(specFiles []string) ([]*spec.Swagger, error) {
   	tmpDir, err := ioutil.TempDir("", "pulsar-spec")
   	if err != nil {
   		return nil, err
   	}
   	defer os.RemoveAll(tmpDir)
   	specs := make([]*spec.Swagger, len(specFiles))
   	for i, specFile := range specFiles {
   		url := fmt.Sprintf(`https://pulsar.apache.org/swagger/%s/%s`, pulsarVersion, specFile)
   		log.Printf("downloading %s", url)
   		fileName, err := getURLToFile(tmpDir, url)
   		if err != nil {
   			return nil, fmt.Errorf("error retrieving %s: %w", url, err)
   		}
   		doc, err := loads.Spec(fileName)
   		if err != nil {
   			return nil, fmt.Errorf("error loading tempfile: %w", err)
   		}
   		specs[i] = doc.Spec()
   	}
   	return specs, nil
   }
   
   // map of meaningful operation names for storage in case we want to base a heuristic off of it (unused)
   var opNames = []string{"GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"}
   
   func getOpConflicts(s *spec.Swagger) ([]*conflictList, error) {
   	conflicts := make(opConflicts)
   	for pathName, path := range s.Paths.Paths {
   		ops := []*spec.Operation{path.Get, path.Post, path.Put, path.Delete, path.Patch, path.Options, path.Head}
   		for i, op := range ops {
   			if op != nil {
   				if op.ID == "" {
   					return nil, fmt.Errorf("%s method for route %s has empty opID", opNames[i], pathName)
   				}
   				if conflicts[op.ID] == nil {
   					conflicts[op.ID] = new(conflictList)
   				}
   				conflicts[op.ID].conflicts = append(conflicts[op.ID].conflicts, &opConflict{
   					op:     op,
   					route:  pathName,
   					method: opNames[i],
   				})
   			}
   		}
   	}
   	for opName, clist := range conflicts {
   		if len(clist.conflicts) < 2 {
   			delete(conflicts, opName)
   		}
   	}
   	cl := make([]*conflictList, 0, len(conflicts))
   	for _, clist := range conflicts {
   		cl = append(cl, clist)
   	}
   	return cl, nil
   }
   
   // apply heuristics to resolve duplicate operation names. usually, one operation name doesn't need changed, so retain one, but if we find more than one we don't know how to rename, return an error
   func resolveDuplicatOpIDs(s *spec.Swagger) error {
   	opConflicts, err := getOpConflicts(s)
   	if err != nil {
   		return err
   	}
   	for _, clist := range opConflicts {
   		for _, c := range clist.conflicts {
   			// attempt to apply resolution rules
   			routeParts := strings.Split(c.route, "/")
   			switch {
   			case strings.HasPrefix(c.route, "/non-persistent"):
   				c.op.ID += "NP"
   			case strings.HasPrefix(c.route, "/namespace"):
   				parts := camelcase.Split(c.op.ID)
   				parts[0] += "NameSpace"
   				c.op.ID = strings.Join(parts, "")
   			case strings.HasSuffix(c.route, "{version}"):
   				c.op.ID += "Version"
   			case routeParts[len(routeParts)-2] == "{instanceId}":
   				c.op.ID += "Instance"
   			default:
   				clist.unchanged = append(clist.unchanged, c)
   				if len(clist.unchanged) > 1 {
   					var tooMany strings.Builder
   					tooMany.WriteString(fmt.Sprintf("Found more than one OP conflict I couldn't handle for opID [%s]: ", c.op.ID))
   					for _, uc := range clist.unchanged {
   						tooMany.WriteString(
   							fmt.Sprintf("%s-%s", uc.method, uc.route),
   						)
   					}
   					return fmt.Errorf(tooMany.String())
   				}
   			}
   		}
   	}
   	return nil
   }
   
   // retrieve the swagger stored at URL to a temporary file. Necessary because loads.Spec requires a file
   func getURLToFile(tmpDir string, url string) (filename string, err error) {
   	file, err := os.Create(filepath.Join(tmpDir, url[strings.LastIndex(url, `/`)+1:]))
   	if err != nil {
   		return "", err
   	}
   
   	defer func() {
   		if err != nil {
   			os.Remove(file.Name())
   		}
   	}()
   
   	var resp *http.Response
   	resp, err = http.Get(url)
   	if err != nil {
   		return "", err
   	}
   	if resp.StatusCode != http.StatusOK {
   		return "", fmt.Errorf("code %d when getting %s", resp.StatusCode, url)
   	}
   	if _, err := io.Copy(file, resp.Body); err != nil {
   		return "", err
   	}
   	return file.Name(), file.Close()
   }
   
   const adminPrefix = `/admin`
   
   // remove the version suffix from the admin path and prepend to all routes so all routes can live together in the combined spec under /admin
   func fixPaths(s *spec.Swagger) error {
   	s.Info.Version = pulsarVersion
   	if !strings.HasPrefix(s.BasePath, adminPrefix) {
   		return fmt.Errorf("basePath does not have required %q prefix: %q", adminPrefix, s.BasePath)
   	}
   	newPrefix := s.BasePath[len(adminPrefix):]
   	s.BasePath = adminPrefix
   	for pathName, path := range s.Paths.Paths {
   		newPathName := newPrefix + pathName
   		if _, ok := s.Paths.Paths[newPathName]; ok {
   			return fmt.Errorf("new path %q already exists", newPathName)
   		}
   		s.Paths.Paths[newPathName] = path
   		delete(s.Paths.Paths, pathName)
   	}
   	return nil
   }
   
   func ensureNoDuplicateDefinitions(specs []*spec.Swagger) error {
   	// create a map of serialized definitions to ensure that definitions of the same name are in fact the same and will preserve refs
   	serializedDefs := map[string][]byte{}
   	for _, s := range specs {
   		for defName, defSchema := range s.Definitions {
   			b, err := json.Marshal(defSchema)
   			if err != nil {
   				return fmt.Errorf("error serializing defininition schema [%s]: %w", defName, err)
   			}
   			if schemaBytes, ok := serializedDefs[defName]; ok {
   				if !bytes.Equal(schemaBytes, b) {
   					return fmt.Errorf("definitions of %q are different", defName)
   				}
   				// remove the duplicate from the spec
   				delete(s.Definitions, defName)
   				continue
   			}
   			// otherwise store the serialized definition
   			serializedDefs[defName] = b
   		}
   	}
   	return nil
   }
   ```


----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org