You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mynewt.apache.org by an...@apache.org on 2022/03/14 18:02:34 UTC
[mynewt-newt] branch master updated (d611573 -> f793cce)
This is an automated email from the ASF dual-hosted git repository.
andk pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mynewt-newt.git.
from d611573 Add const syscfg values
new 3b8ea4d Add dependencies for sysinit functions
new 1a8f2c4 Print sysinit dependencis in sysinit config output
new f793cce Add sysinit graph output in DOT format
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
newt/cli/target_cfg_cmds.go | 72 +++++++++++++++++++-
newt/pkg/localpackage.go | 4 +-
newt/stage/stage.go | 58 +++++++++++++++--
newt/sysinit/sysinit.go | 155 +++++++++++++++++++++++++++++++++++++++++++-
newt/val/valsetting.go | 4 ++
5 files changed, 282 insertions(+), 11 deletions(-)
[mynewt-newt] 01/03: Add dependencies for sysinit functions
Posted by an...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
andk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mynewt-newt.git
commit 3b8ea4dbd89063bf68c8491adfa8b04c7e51226b
Author: Andrzej Kaczmarek <an...@codecoup.pl>
AuthorDate: Mon Mar 7 10:07:31 2022 +0100
Add dependencies for sysinit functions
Currently it's only possible to specify an arbitrary stage number for
sysinit function. This can be a bit troublesome if we need to assure
some packages are initialized in specific order since it required to
adjust stages for all relevant packages. Also some packages have stages
defined via syscfg (which is horrible) which means they can be changed
and the whole setup breaks.
This patch allows to order sysinit by dependencies between functions.
New syntax is allowed in addition to stage number:
- $after:func_name
- $before:func_name
The former will ensure that function is called after "func_name", and
the latter - after "func_name".
Each sysinit function shall specify one stage number or any number of
dependencies - those cannot be mixed. Some examples of valid sysinit
definitions:
(1)
func_1: 100
(2)
func_1: $before:func_2
func_2: $after:func_1
(3)
func_1:
- $before:func_2
- $after:func_3
func_2: $before:func_4,$before:func_5
Note that (2) is ok even though both functions depend on each other,
since effectively both dependencies are the same.
Multiple dependencies can be specified as a list or as a comma-separated
string (3).
An error will be raised if circular dependency is found when resolving
order of functions.
---
newt/pkg/localpackage.go | 4 +-
newt/stage/stage.go | 57 ++++++++++++++++--
newt/sysinit/sysinit.go | 152 ++++++++++++++++++++++++++++++++++++++++++++++-
newt/val/valsetting.go | 4 ++
4 files changed, 208 insertions(+), 9 deletions(-)
diff --git a/newt/pkg/localpackage.go b/newt/pkg/localpackage.go
index f1e3362..a68cbf2 100644
--- a/newt/pkg/localpackage.go
+++ b/newt/pkg/localpackage.go
@@ -367,9 +367,9 @@ func (pkg *LocalPackage) Load() error {
}
func (pkg *LocalPackage) InitFuncs(
- settings map[string]string) map[string]string {
+ settings map[string]string) map[string]interface{} {
- vals, err := pkg.PkgY.GetValStringMapString("pkg.init", settings)
+ vals, err := pkg.PkgY.GetValStringMap("pkg.init", settings)
util.OneTimeWarningError(err)
return vals
}
diff --git a/newt/stage/stage.go b/newt/stage/stage.go
index dce33c3..b028e44 100644
--- a/newt/stage/stage.go
+++ b/newt/stage/stage.go
@@ -47,6 +47,9 @@ type StageFunc struct {
ReturnType string
ArgList string
Pkg *pkg.LocalPackage
+ Deps []*StageFunc
+ Resolved bool
+ Resolving bool
}
func NewStageFunc(name string, textVal string,
@@ -73,6 +76,36 @@ func NewStageFunc(name string, textVal string,
return sf, nil
}
+func NewStageFuncMultiDeps(name string, stageDeps []string,
+ p *pkg.LocalPackage, cfg *syscfg.Cfg) (StageFunc, error) {
+ sf := StageFunc{
+ Name: name,
+ Pkg: p,
+ Stage: val.ValSetting{},
+ }
+
+ for _, stageDep := range stageDeps {
+ s := strings.TrimPrefix(stageDep, "$before:")
+ if s != stageDep {
+ sf.Stage.Befores = append(sf.Stage.Befores, s)
+ } else {
+ s = strings.TrimPrefix(stageDep, "$after:")
+ if s != stageDep {
+ sf.Stage.Afters = append(sf.Stage.Afters, s)
+ } else {
+ return StageFunc{}, util.FmtNewtError("Invalid setting: \"%s: %s\"; "+
+ "value should specify a $before or $after dependency.", name, stageDep)
+ }
+ }
+ }
+
+ return sf, nil
+}
+
+func ValIsDep(textVal string) bool {
+ return strings.HasPrefix(textVal, "$before:") || strings.HasPrefix(textVal, "$after:")
+}
+
// SortStageFuncs performs an in-place sort of the provided StageFunc slice.
func SortStageFuncs(sfs []StageFunc, entryType string) {
sort.Slice(sfs, func(i int, j int) bool {
@@ -138,21 +171,37 @@ func WriteCalls(sortedFuncs []StageFunc, argList string, w io.Writer) {
for i, f := range sortedFuncs {
intStage, _ := f.Stage.IntVal()
+ noStage := len(f.Stage.Afters) > 0 || len(f.Stage.Befores) > 0
if intStage != prevStage {
prevStage = intStage
dupCount = 0
- if i != 0 {
+ if noStage {
fmt.Fprintf(w, "\n")
+ } else {
+ if i != 0 {
+ fmt.Fprintf(w, "\n")
+ }
+ fmt.Fprintf(w, " /*** Stage %d */\n", intStage)
}
- fmt.Fprintf(w, " /*** Stage %d */\n", intStage)
} else {
dupCount += 1
}
- fmt.Fprintf(w, " /* %d.%d: %s (%s) */\n",
- intStage, dupCount, f.Name, f.Pkg.Name())
+ if noStage {
+ for _, s := range f.Stage.Afters {
+ fmt.Fprintf(w, " /* [$after:%s]: %s (%s) */\n",
+ s, f.Name, f.Pkg.Name())
+ }
+ for _, s := range f.Stage.Befores {
+ fmt.Fprintf(w, " /* [$before:%s]: %s (%s) */\n",
+ s, f.Name, f.Pkg.Name())
+ }
+ } else {
+ fmt.Fprintf(w, " /* %d.%d: %s (%s) */\n",
+ intStage, dupCount, f.Name, f.Pkg.Name())
+ }
fmt.Fprintf(w, " %s(%s);\n", f.Name, argList)
}
}
diff --git a/newt/sysinit/sysinit.go b/newt/sysinit/sysinit.go
index dd270de..73c822d 100644
--- a/newt/sysinit/sysinit.go
+++ b/newt/sysinit/sysinit.go
@@ -22,7 +22,11 @@ package sysinit
import (
"bytes"
"fmt"
+ "github.com/spf13/cast"
"io"
+ "mynewt.apache.org/newt/util"
+ "sort"
+ "strings"
"mynewt.apache.org/newt/newt/newtutil"
"mynewt.apache.org/newt/newt/pkg"
@@ -45,8 +49,30 @@ type SysinitCfg struct {
func (scfg *SysinitCfg) readOnePkg(lpkg *pkg.LocalPackage, cfg *syscfg.Cfg) {
settings := cfg.AllSettingsForLpkg(lpkg)
initMap := lpkg.InitFuncs(settings)
- for name, stageStr := range initMap {
- sf, err := stage.NewStageFunc(name, stageStr, lpkg, cfg)
+ for name, stageDef := range initMap {
+ var sf stage.StageFunc
+ var err error
+
+ switch stageDef.(type) {
+ default:
+ stageStr := cast.ToString(stageDef)
+
+ if stage.ValIsDep(stageStr) {
+ stageDeps := strings.Split(stageStr, ",")
+ sf, err = stage.NewStageFuncMultiDeps(name, stageDeps, lpkg, cfg)
+ } else {
+ sf, err = stage.NewStageFunc(name, stageStr, lpkg, cfg)
+ }
+ case []interface{}:
+ var stageDeps []string
+
+ for _, stageDepIf := range stageDef.([]interface{}) {
+ stageDeps = append(stageDeps, cast.ToString(stageDepIf))
+ }
+
+ sf, err = stage.NewStageFuncMultiDeps(name, stageDeps, lpkg, cfg)
+ }
+
if err != nil {
scfg.InvalidSettings = append(scfg.InvalidSettings, err.Error())
} else {
@@ -71,6 +97,114 @@ func (scfg *SysinitCfg) detectConflicts() {
}
}
+func stageFuncResolve(sfRef *stage.StageFunc, stack *[]stage.StageFunc) error {
+ if sfRef.Resolved {
+ return nil
+ }
+ if sfRef.Resolving {
+ return util.FmtNewtError("Circular dependency detected while resolving \"%s (%s)\".",
+ sfRef.Name, sfRef.Pkg.FullName())
+ }
+
+ sfRef.Resolving = true
+ for _, sfDepRef := range sfRef.Deps {
+ err := stageFuncResolve(sfDepRef, stack)
+ if err != nil {
+ return err
+ }
+ }
+ sfRef.Resolving = false
+
+ sfRef.Resolved = true
+ *stack = append([]stage.StageFunc{*sfRef}, *stack...)
+
+ return nil
+}
+
+func ResolveStageFuncsOrder(sfs []stage.StageFunc) ([]stage.StageFunc, error) {
+ var nodes []*stage.StageFunc
+ var nodesQ []*stage.StageFunc
+ nodesByStage := make(map[int][]*stage.StageFunc)
+ nodesByName := make(map[string]*stage.StageFunc)
+
+ for idx, _ := range sfs {
+ sfRef := &sfs[idx]
+ nodesByName[sfRef.Name] = sfRef
+ if len(sfRef.Stage.Befores) == 0 && len(sfRef.Stage.Afters) == 0 {
+ stage, _ := sfRef.Stage.IntVal()
+ nodesByStage[stage] = append(nodesByStage[stage], sfRef)
+ nodes = append(nodes, sfRef)
+ } else {
+ nodesQ = append(nodesQ, sfRef)
+ }
+ }
+
+ // Put nodes without stages first, so they are resolved and put to
+ // stack first - we do not want them to precede all nodes with stages.
+ // While technically correct, it's better not to start sysinit with
+ // nodes that do not have any stage since this will likely happen
+ // before os init packages.
+ nodes = append(nodesQ, nodes...)
+
+ var stages []int
+ for stage := range nodesByStage {
+ stages = append(stages, stage)
+ }
+ sort.Ints(stages)
+
+ // Add implicit dependencies for nodes with stages. We need to add
+ // direct dependencies between each node of stage X to each node of
+ // stage Y to make sure they can be resolved properly and reordered
+ // if needed due to other dependencies.
+ sfsPrev := nodesByStage[stages[0]]
+ stages = stages[1:]
+ for _, stage := range stages {
+ sfsCurr := nodesByStage[stage]
+
+ for _, sfsP := range sfsPrev {
+ for _, sfsC := range sfsCurr {
+ sfsP.Deps = append(sfsP.Deps, sfsC)
+ }
+ }
+
+ sfsPrev = sfsCurr
+ }
+
+ // Now add other dependencies, i.e. $after and $before
+ for _, sf := range nodesQ {
+ for _, depStr := range sf.Stage.Afters {
+ depSf := nodesByName[depStr]
+ if depSf == nil {
+ return []stage.StageFunc{},
+ util.FmtNewtError("Unknown depdendency (\"%s\") for \"%s (%s)\".",
+ depStr, sf.Name, sf.Pkg.FullName())
+ }
+ depSf.Deps = append(depSf.Deps, sf)
+ }
+ for _, depStr := range sf.Stage.Befores {
+ depSf := nodesByName[depStr]
+ if depSf == nil {
+ return []stage.StageFunc{},
+ util.FmtNewtError("Unknown depdendency (\"%s\") for \"%s (%s)\".",
+ depStr, sf.Name, sf.Pkg.FullName())
+ }
+ sf.Deps = append(sf.Deps, depSf)
+ }
+ }
+
+ // Now we can resolve order of functions by sorting dependency graph
+ // topologically. This will also detect circular dependencies.
+ sfs = []stage.StageFunc{}
+ for _, sfRef := range nodes {
+ err := stageFuncResolve(sfRef, &sfs)
+ if err != nil {
+ return []stage.StageFunc{}, err
+ }
+ }
+
+ return sfs, nil
+}
+
func Read(lpkgs []*pkg.LocalPackage, cfg *syscfg.Cfg) SysinitCfg {
scfg := SysinitCfg{
Conflicts: map[string][]stage.StageFunc{},
@@ -81,7 +215,19 @@ func Read(lpkgs []*pkg.LocalPackage, cfg *syscfg.Cfg) SysinitCfg {
}
scfg.detectConflicts()
- stage.SortStageFuncs(scfg.StageFuncs, "sysinit")
+
+ // Don't try to resolve order if there are name conflicts since that
+ // process depends on unique names for each function and could return
+ // other confusing errors.
+ if len(scfg.Conflicts) > 0 {
+ return scfg
+ }
+
+ var err error
+ scfg.StageFuncs, err = ResolveStageFuncsOrder(scfg.StageFuncs)
+ if err != nil {
+ scfg.InvalidSettings = append(scfg.InvalidSettings, err.Error())
+ }
return scfg
}
diff --git a/newt/val/valsetting.go b/newt/val/valsetting.go
index a57d1d1..0d24218 100644
--- a/newt/val/valsetting.go
+++ b/newt/val/valsetting.go
@@ -37,6 +37,10 @@ type ValSetting struct {
// The setting value, after setting references are resolved.
Value string
+
+ // List of $after and $before dependencies
+ Afters []string
+ Befores []string
}
// IntVal Extracts a setting's integer value.
[mynewt-newt] 03/03: Add sysinit graph output in DOT format
Posted by an...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
andk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mynewt-newt.git
commit f793ccec2d849b916d83c9e94263e3e51ed8d24b
Author: Andrzej Kaczmarek <an...@codecoup.pl>
AuthorDate: Mon Mar 7 11:03:42 2022 +0100
Add sysinit graph output in DOT format
This adds "newt target sysinit graphviz" subcommand that outputs sysinit
structure in DOT format which allows to render a graph of dependencies.
The simplest way to create a graph is by using Graphviz and ImageMagick:
$ newt target sysinit graphviz targetname | dot -Tsvg | display
---
newt/cli/target_cfg_cmds.go | 59 +++++++++++++++++++++++++++++++++++++++++++++
newt/stage/stage.go | 1 +
newt/sysinit/sysinit.go | 3 +++
3 files changed, 63 insertions(+)
diff --git a/newt/cli/target_cfg_cmds.go b/newt/cli/target_cfg_cmds.go
index f7a0dc9..98352dd 100644
--- a/newt/cli/target_cfg_cmds.go
+++ b/newt/cli/target_cfg_cmds.go
@@ -575,6 +575,30 @@ func printSysinitBrief(targetName string, scfg sysinit.SysinitCfg) {
printStageBriefTable(scfg.StageFuncs)
}
+func printSysinitGraphviz(targetName string, scfg sysinit.SysinitCfg) {
+ if errText := scfg.ErrorText(); errText != "" {
+ util.StatusMessage(util.VERBOSITY_DEFAULT, "!!! %s\n\n", errText)
+ }
+
+ fmt.Printf("digraph sysinit {\n")
+ for _, sf := range scfg.StageFuncs {
+ if len(sf.Stage.Afters) == 0 && len(sf.Stage.Befores) == 0 {
+ stage, _ := sf.Stage.IntVal()
+ fmt.Printf(" %s [label=\"%s (%d)\"];\n", sf.Name, sf.Name, stage)
+ }
+ for _, depSf := range sf.DepsI {
+ fmt.Printf(" %s -> %s;\n", sf.Name, depSf.Name)
+ }
+ for _, depStr := range sf.Stage.Afters {
+ fmt.Printf(" %s -> %s [label=\"$after:%s\"];\n", depStr, sf.Name, depStr)
+ }
+ for _, depStr := range sf.Stage.Befores {
+ fmt.Printf(" %s -> %s [label=\"$before:%s\"];\n", sf.Name, depStr, depStr)
+ }
+ }
+ fmt.Printf("}\n")
+}
+
func targetSysinitShowCmd(cmd *cobra.Command, args []string) {
if len(args) < 1 {
NewtUsage(cmd,
@@ -621,6 +645,29 @@ func targetSysinitBriefCmd(cmd *cobra.Command, args []string) {
}
}
+func targetSysinitGraphvizCmd(cmd *cobra.Command, args []string) {
+ if len(args) < 1 {
+ NewtUsage(cmd,
+ util.NewNewtError("Must specify target or unittest name"))
+ }
+
+ TryGetProject()
+
+ for i, arg := range args {
+ b, err := TargetBuilderForTargetOrUnittest(arg)
+ if err != nil {
+ NewtUsage(cmd, err)
+ }
+
+ res := targetBuilderConfigResolve(b)
+ printSysinitGraphviz(b.GetTarget().Name(), res.SysinitCfg)
+
+ if i < len(args)-1 {
+ util.StatusMessage(util.VERBOSITY_DEFAULT, "\n")
+ }
+ }
+}
+
func printSysdownCfg(targetName string, scfg sysdown.SysdownCfg) {
util.StatusMessage(util.VERBOSITY_DEFAULT, "Sysdown config for %s:\n",
targetName)
@@ -939,6 +986,18 @@ func targetCfgCmdAll() []*cobra.Command {
return append(targetList(), unittestList()...)
})
+ sysinitGraphvizCmd := &cobra.Command{
+ Use: "graphviz <target> [target...]",
+ Short: "Print a graph representation of target's sysinit configuration (DOT format)",
+ Long: "Print a graph representation of target's sysinit configuration (DOT format)",
+ Run: targetSysinitGraphvizCmd,
+ }
+
+ sysinitCmd.AddCommand(sysinitGraphvizCmd)
+ AddTabCompleteFn(sysinitGraphvizCmd, func() []string {
+ return append(targetList(), unittestList()...)
+ })
+
sysdownHelpText := "View a target's sysdown configuration"
sysdownCmd := &cobra.Command{
diff --git a/newt/stage/stage.go b/newt/stage/stage.go
index b028e44..4e39ca2 100644
--- a/newt/stage/stage.go
+++ b/newt/stage/stage.go
@@ -48,6 +48,7 @@ type StageFunc struct {
ArgList string
Pkg *pkg.LocalPackage
Deps []*StageFunc
+ DepsI []*StageFunc
Resolved bool
Resolving bool
}
diff --git a/newt/sysinit/sysinit.go b/newt/sysinit/sysinit.go
index 73c822d..efa226a 100644
--- a/newt/sysinit/sysinit.go
+++ b/newt/sysinit/sysinit.go
@@ -164,6 +164,9 @@ func ResolveStageFuncsOrder(sfs []stage.StageFunc) ([]stage.StageFunc, error) {
for _, sfsP := range sfsPrev {
for _, sfsC := range sfsCurr {
sfsP.Deps = append(sfsP.Deps, sfsC)
+ // Keep separate list of implicit dependencies
+ // This is only used for Graphviz output
+ sfsP.DepsI = append(sfsP.DepsI, sfsC)
}
}
[mynewt-newt] 02/03: Print sysinit dependencis in sysinit config output
Posted by an...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
andk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mynewt-newt.git
commit 1a8f2c4133452d949eac88f5b0c436dc5b3c65a7
Author: Andrzej Kaczmarek <an...@codecoup.pl>
AuthorDate: Mon Mar 7 10:48:11 2022 +0100
Print sysinit dependencis in sysinit config output
---
newt/cli/target_cfg_cmds.go | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/newt/cli/target_cfg_cmds.go b/newt/cli/target_cfg_cmds.go
index 8ea1ba0..f7a0dc9 100644
--- a/newt/cli/target_cfg_cmds.go
+++ b/newt/cli/target_cfg_cmds.go
@@ -483,8 +483,17 @@ func printStage(sf stage.StageFunc) {
util.StatusMessage(util.VERBOSITY_DEFAULT, "%s:\n", sf.Name)
util.StatusMessage(util.VERBOSITY_DEFAULT, " Package: %s\n",
sf.Pkg.FullName())
- util.StatusMessage(util.VERBOSITY_DEFAULT, " Stage: %s\n",
- valSettingString(sf.Stage))
+ if len(sf.Stage.Afters) > 0 || len(sf.Stage.Befores) > 0 {
+ for _, s := range sf.Stage.Afters {
+ util.StatusMessage(util.VERBOSITY_DEFAULT, " After: %s\n", s)
+ }
+ for _, s := range sf.Stage.Befores {
+ util.StatusMessage(util.VERBOSITY_DEFAULT, " Before: %s\n", s)
+ }
+ } else {
+ util.StatusMessage(util.VERBOSITY_DEFAULT, " Stage: %s\n",
+ valSettingString(sf.Stage))
+ }
}
func printStageBriefOne(sf stage.StageFunc,