You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mynewt.apache.org by cc...@apache.org on 2017/01/05 03:18:02 UTC
incubator-mynewt-newt git commit: MYNEWT-537 newt - Multithreaded
compiles
Repository: incubator-mynewt-newt
Updated Branches:
refs/heads/develop 205009dff -> c1dfedcd1
MYNEWT-537 newt - Multithreaded compiles
To specify the number of concurrent threads, use the "-j" option:
newt -j 5 build bleprph-nrf52dk
newt -j 5 create-image bleprph-nrf52dk 1.2.3.4
newt -j 5 run bleprph-nrf52dk 0
Project: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/commit/c1dfedcd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/tree/c1dfedcd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/diff/c1dfedcd
Branch: refs/heads/develop
Commit: c1dfedcd1595882ddd20f042a16fc2ed89260b46
Parents: 205009d
Author: Christopher Collins <cc...@apache.org>
Authored: Wed Jan 4 12:44:46 2017 -0800
Committer: Christopher Collins <cc...@apache.org>
Committed: Wed Jan 4 19:18:05 2017 -0800
----------------------------------------------------------------------
newt/builder/build.go | 187 +++++++++++++++++++-----
newt/cli/run_cmds.go | 2 +-
newt/newt.go | 5 +
newt/newtutil/newtutil.go | 1 +
newt/toolchain/compiler.go | 311 +++++++++++++++++++++-------------------
5 files changed, 321 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/builder/build.go
----------------------------------------------------------------------
diff --git a/newt/builder/build.go b/newt/builder/build.go
index 6e2fbdb..b41d69d 100644
--- a/newt/builder/build.go
+++ b/newt/builder/build.go
@@ -27,6 +27,7 @@ import (
log "github.com/Sirupsen/logrus"
"mynewt.apache.org/newt/newt/image"
+ "mynewt.apache.org/newt/newt/newtutil"
"mynewt.apache.org/newt/newt/pkg"
"mynewt.apache.org/newt/newt/repo"
"mynewt.apache.org/newt/newt/symbol"
@@ -144,12 +145,12 @@ func pkgTypeConflictErr(p1 *BuildPackage, p2 *BuildPackage) error {
// Recursively compiles all the .c and .s files in the specified directory.
// Architecture-specific files are also compiled.
-func buildDir(srcDir string, c *toolchain.Compiler, arch string,
- ignDirs []string) error {
+func collectCompileEntriesDir(srcDir string, c *toolchain.Compiler,
+ arch string, ignDirs []string) ([]toolchain.CompilerJob, error) {
// Quietly succeed if the source directory doesn't exist.
if util.NodeNotExist(srcDir) {
- return nil
+ return nil, nil
}
util.StatusMessage(util.VERBOSITY_VERBOSE,
@@ -161,23 +162,32 @@ func buildDir(srcDir string, c *toolchain.Compiler, arch string,
// Ignore architecture-specific source files for now. Use a temporary
// string slice here so that the "arch" directory is not ignored in the
// subsequent architecture-specific compile phase.
- if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_C,
- append(ignDirs, "arch")); err != nil {
+ ignDirsArch := append(ignDirs, "arch")
- return err
+ entries := []toolchain.CompilerJob{}
+
+ subEntries, err := c.RecursiveCollectEntries(toolchain.COMPILER_TYPE_C,
+ ignDirsArch)
+ if err != nil {
+ return nil, err
}
+ entries = append(entries, subEntries...)
// Compile CPP files
- if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_CPP,
- append(ignDirs, "arch")); err != nil {
- return err
+ subEntries, err = c.RecursiveCollectEntries(toolchain.COMPILER_TYPE_CPP,
+ ignDirsArch)
+ if err != nil {
+ return nil, err
}
+ entries = append(entries, subEntries...)
// Copy in pre-compiled library files
- if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_ARCHIVE,
- append(ignDirs, "arch")); err != nil {
- return err
+ subEntries, err = c.RecursiveCollectEntries(
+ toolchain.COMPILER_TYPE_ARCHIVE, ignDirsArch)
+ if err != nil {
+ return nil, err
}
+ entries = append(entries, subEntries...)
archDir := srcDir + "/arch/" + arch + "/"
if util.NodeExist(archDir) {
@@ -187,33 +197,39 @@ func buildDir(srcDir string, c *toolchain.Compiler, arch string,
c.SetSrcDir(archDir)
// Compile C source.
- if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_C,
- ignDirs); err != nil {
-
- return err
+ subEntries, err = c.RecursiveCollectEntries(
+ toolchain.COMPILER_TYPE_C, ignDirs)
+ if err != nil {
+ return nil, err
}
+ entries = append(entries, subEntries...)
// Compile CPP source
- if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_CPP,
- ignDirs); err != nil {
- return err
+ subEntries, err = c.RecursiveCollectEntries(
+ toolchain.COMPILER_TYPE_CPP, ignDirs)
+ if err != nil {
+ return nil, err
}
+ entries = append(entries, subEntries...)
// Compile assembly source (only architecture-specific).
- if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_ASM,
- ignDirs); err != nil {
-
- return err
+ subEntries, err = c.RecursiveCollectEntries(
+ toolchain.COMPILER_TYPE_ASM, ignDirs)
+ if err != nil {
+ return nil, err
}
+ entries = append(entries, subEntries...)
// Copy in pre-compiled library files
- if err := c.RecursiveCompile(toolchain.COMPILER_TYPE_ARCHIVE,
- ignDirs); err != nil {
- return err
+ subEntries, err = c.RecursiveCollectEntries(
+ toolchain.COMPILER_TYPE_ARCHIVE, ignDirs)
+ if err != nil {
+ return nil, err
}
+ entries = append(entries, subEntries...)
}
- return nil
+ return entries, nil
}
func (b *Builder) newCompiler(bpkg *BuildPackage,
@@ -238,11 +254,12 @@ func (b *Builder) newCompiler(bpkg *BuildPackage,
return c, nil
}
-// Compiles and archives a package.
-func (b *Builder) buildPackage(bpkg *BuildPackage) error {
+func (b *Builder) collectCompileEntriesBpkg(bpkg *BuildPackage) (
+ []toolchain.CompilerJob, error) {
+
c, err := b.newCompiler(bpkg, b.PkgBinDir(bpkg))
if err != nil {
- return err
+ return nil, err
}
srcDirs := []string{}
@@ -251,7 +268,7 @@ func (b *Builder) buildPackage(bpkg *BuildPackage) error {
for _, relDir := range bpkg.SourceDirectories {
dir := bpkg.BasePath() + "/" + relDir
if util.NodeNotExist(dir) {
- return util.NewNewtError(fmt.Sprintf(
+ return nil, util.NewNewtError(fmt.Sprintf(
"Specified source directory %s, does not exist.",
dir))
}
@@ -261,22 +278,33 @@ func (b *Builder) buildPackage(bpkg *BuildPackage) error {
srcDir := bpkg.BasePath() + "/src"
if util.NodeNotExist(srcDir) {
// Nothing to compile.
- return nil
+ return nil, nil
}
srcDirs = append(srcDirs, srcDir)
}
+ entries := []toolchain.CompilerJob{}
for _, dir := range srcDirs {
- if err = buildDir(dir, c, b.targetBuilder.bspPkg.Arch, nil); err != nil {
- return err
+ subEntries, err := collectCompileEntriesDir(dir, c,
+ b.targetBuilder.bspPkg.Arch, nil)
+ if err != nil {
+ return nil, err
}
+
+ entries = append(entries, subEntries...)
}
+ return entries, nil
+}
+
+func (b *Builder) createArchive(c *toolchain.Compiler,
+ bpkg *BuildPackage) error {
+
// Create a static library ("archive").
c.SetSrcDir(bpkg.RelativePath())
archiveFile := b.ArchivePath(bpkg)
- if err = c.CompileArchive(archiveFile); err != nil {
+ if err := c.CompileArchive(archiveFile); err != nil {
return err
}
@@ -427,16 +455,103 @@ func (b *Builder) addSysinitBpkg() (*BuildPackage, error) {
return b.addPackage(lpkg)
}
+// Runs build jobs while any remain. On failure, signals the other workers to
+// stop via the stop channel. On error, the error object is signaled via the
+// results channel. On successful completion, nil is signaled via the results
+// channel.
+func buildWorker(
+ id int,
+ jobs <-chan toolchain.CompilerJob,
+ stop chan struct{},
+ results chan error) {
+
+ // Execute each job until failure or until a stop is signalled.
+ for {
+ select {
+ case s := <-stop:
+ // Re-enqueue the stop signal for the other routines.
+ stop <- s
+
+ // Terminate this go routine.
+ results <- nil
+ return
+
+ case j := <-jobs:
+ if err := toolchain.RunJob(j); err != nil {
+ // Stop the other routines.
+ stop <- struct{}{}
+
+ // Report the error back to the master thread and terminate.
+ results <- err
+ return
+ }
+
+ default:
+ // Terminate this go routine.
+ results <- nil
+ return
+ }
+ }
+}
+
func (b *Builder) Build() error {
b.CleanArtifacts()
// Build the packages alphabetically to ensure a consistent order.
bpkgs := b.sortedBuildPackages()
+ // Calculate the list of jobs. Each record represents a single file that
+ // needs to be compiled.
+ entries := []toolchain.CompilerJob{}
+ bpkgCompilerMap := map[*BuildPackage]*toolchain.Compiler{}
for _, bpkg := range bpkgs {
- if err := b.buildPackage(bpkg); err != nil {
+ subEntries, err := b.collectCompileEntriesBpkg(bpkg)
+ if err != nil {
return err
}
+ entries = append(entries, subEntries...)
+
+ if len(subEntries) > 0 {
+ bpkgCompilerMap[bpkg] = subEntries[0].Compiler
+ }
+ }
+
+ // Build each file in parallel.
+ jobs := make(chan toolchain.CompilerJob, len(entries))
+ defer close(jobs)
+
+ stop := make(chan struct{}, 1)
+ defer close(stop)
+
+ errors := make(chan error, newtutil.NewtNumJobs)
+ defer close(errors)
+
+ for _, entry := range entries {
+ jobs <- entry
+ }
+
+ for i := 0; i < newtutil.NewtNumJobs; i++ {
+ go buildWorker(i, jobs, stop, errors)
+ }
+
+ var err error
+ for i := 0; i < newtutil.NewtNumJobs; i++ {
+ subErr := <-errors
+ if err == nil && subErr != nil {
+ err = subErr
+ }
+ }
+ if err != nil {
+ return err
+ }
+
+ for _, bpkg := range bpkgs {
+ c := bpkgCompilerMap[bpkg]
+ if c != nil {
+ if err := b.createArchive(c, bpkg); err != nil {
+ return err
+ }
+ }
}
return nil
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/cli/run_cmds.go
----------------------------------------------------------------------
diff --git a/newt/cli/run_cmds.go b/newt/cli/run_cmds.go
index 2265d6c..55d7bbc 100644
--- a/newt/cli/run_cmds.go
+++ b/newt/cli/run_cmds.go
@@ -98,7 +98,7 @@ func AddRunCommands(cmd *cobra.Command) {
Run: runRunCmd,
}
- runCmd.PersistentFlags().StringVarP(&extraJtagCmd, "extrajtagcmd", "j", "",
+ runCmd.PersistentFlags().StringVarP(&extraJtagCmd, "extrajtagcmd", "", "",
"extra commands to send to JTAG software")
runCmd.PersistentFlags().BoolVarP(&noGDB_flag, "noGDB", "n", false,
"don't start GDB from command line")
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/newt.go
----------------------------------------------------------------------
diff --git a/newt/newt.go b/newt/newt.go
index e7a4ab0..8395db6 100644
--- a/newt/newt.go
+++ b/newt/newt.go
@@ -36,6 +36,7 @@ var newtSilent bool
var newtQuiet bool
var newtVerbose bool
var newtLogFile string
+var newtNumJobs int
func newtCmd() *cobra.Command {
newtHelpText := cli.FormatHelp(`Newt allows you to create your own embedded
@@ -79,6 +80,8 @@ func newtCmd() *cobra.Command {
if err != nil {
cli.NewtUsage(nil, err)
}
+
+ newtutil.NewtNumJobs = newtNumJobs
},
Run: func(cmd *cobra.Command, args []string) {
cmd.Help()
@@ -95,6 +98,8 @@ func newtCmd() *cobra.Command {
"WARN", "Log level")
newtCmd.PersistentFlags().StringVarP(&newtLogFile, "outfile", "o",
"", "Filename to tee output to")
+ newtCmd.PersistentFlags().IntVarP(&newtNumJobs, "jobs", "j",
+ 1, "Number of concurrent build jobs")
versHelpText := cli.FormatHelp(`Display the Newt version number.`)
versHelpEx := " newt version"
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/newtutil/newtutil.go
----------------------------------------------------------------------
diff --git a/newt/newtutil/newtutil.go b/newt/newtutil/newtutil.go
index 0182514..824e3af 100644
--- a/newt/newtutil/newtutil.go
+++ b/newt/newtutil/newtutil.go
@@ -36,6 +36,7 @@ import (
var NewtVersionStr string = "Apache Newt (incubating) version: 1.0.0-dev"
var NewtBlinkyTag string = "develop"
+var NewtNumJobs int
const NEWTRC_DIR string = ".newt"
const REPOS_FILENAME string = "repos.yml"
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c1dfedcd/newt/toolchain/compiler.go
----------------------------------------------------------------------
diff --git a/newt/toolchain/compiler.go b/newt/toolchain/compiler.go
index 710243b..e7d962e 100644
--- a/newt/toolchain/compiler.go
+++ b/newt/toolchain/compiler.go
@@ -30,6 +30,7 @@ import (
"sort"
"strconv"
"strings"
+ "sync"
"time"
log "github.com/Sirupsen/logrus"
@@ -60,9 +61,13 @@ type CompilerInfo struct {
}
type Compiler struct {
- ObjPathList map[string]bool
+ objPathList map[string]bool
LinkerScripts []string
+ // Needs to be locked whenever a mutable field in this struct is accessed
+ // during a build. Currently, objPathList is the only such member.
+ mutex *sync.Mutex
+
depTracker DepTracker
ccPath string
cppPath string
@@ -93,6 +98,12 @@ type Compiler struct {
extraDeps []string
}
+type CompilerJob struct {
+ Filename string
+ Compiler *Compiler
+ CompilerType int
+}
+
func NewCompilerInfo() *CompilerInfo {
ci := &CompilerInfo{}
ci.Includes = []string{}
@@ -177,7 +188,8 @@ func NewCompiler(compilerDir string, dstDir string,
buildProfile string) (*Compiler, error) {
c := &Compiler{
- ObjPathList: map[string]bool{},
+ mutex: &sync.Mutex{},
+ objPathList: map[string]bool{},
baseDir: project.GetProject().BasePath,
srcDir: "",
dstDir: dstDir,
@@ -292,7 +304,10 @@ func (c *Compiler) AddDeps(depFilenames ...string) {
// executable.
func (c *Compiler) SkipSourceFile(srcFile string) error {
objPath := c.dstFilePath(srcFile) + ".o"
- c.ObjPathList[filepath.ToSlash(objPath)] = true
+
+ c.mutex.Lock()
+ c.objPathList[filepath.ToSlash(objPath)] = true
+ c.mutex.Unlock()
// Update the dependency tracker with the object file's modification time.
// This is necessary later for determining if the library / executable
@@ -316,7 +331,7 @@ func (c *Compiler) includesStrings() []string {
tokens := make([]string, len(includes))
for i, s := range includes {
- s = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(s)), c.baseDir + "/")
+ s = strings.TrimPrefix(filepath.ToSlash(filepath.Clean(s)), c.baseDir+"/")
tokens[i] = "-I" + s
}
@@ -344,7 +359,7 @@ func (c *Compiler) depsString() string {
}
func (c *Compiler) dstFilePath(srcPath string) string {
- relSrcPath := strings.TrimPrefix(filepath.ToSlash(srcPath), c.srcDir + "/")
+ relSrcPath := strings.TrimPrefix(filepath.ToSlash(srcPath), c.srcDir+"/")
relDstPath := strings.TrimSuffix(relSrcPath, filepath.Ext(srcPath))
dstPath := fmt.Sprintf("%s/%s", c.dstDir, relDstPath)
return dstPath
@@ -382,7 +397,7 @@ func (c *Compiler) CompileFileCmd(file string, compilerType int) (
return nil, util.NewNewtError("Unknown compiler type")
}
- srcPath := strings.TrimPrefix(file, c.baseDir + "/")
+ srcPath := strings.TrimPrefix(file, c.baseDir+"/")
cmd := []string{cmdName}
cmd = append(cmd, flags...)
cmd = append(cmd, c.includesStrings()...)
@@ -406,7 +421,7 @@ func (c *Compiler) GenDepsForFile(file string) error {
os.MkdirAll(depDir, 0755)
}
- srcPath := strings.TrimPrefix(file, c.baseDir + "/")
+ srcPath := strings.TrimPrefix(file, c.baseDir+"/")
cmd := []string{c.ccPath}
cmd = append(cmd, c.cflagsStrings()...)
cmd = append(cmd, c.includesStrings()...)
@@ -483,14 +498,16 @@ func (c *Compiler) CompileFile(file string, compilerType int) error {
os.MkdirAll(objDir, 0755)
}
- c.ObjPathList[filepath.ToSlash(objPath)] = true
+ c.mutex.Lock()
+ c.objPathList[filepath.ToSlash(objPath)] = true
+ c.mutex.Unlock()
cmd, err := c.CompileFileCmd(file, compilerType)
if err != nil {
return err
}
- srcPath := strings.TrimPrefix(file, c.baseDir + "/")
+ srcPath := strings.TrimPrefix(file, c.baseDir+"/")
switch compilerType {
case COMPILER_TYPE_C:
util.StatusMessage(util.VERBOSITY_DEFAULT, "Compiling %s\n", srcPath)
@@ -528,79 +545,71 @@ func (c *Compiler) shouldIgnoreFile(file string) bool {
return false
}
-// Compiles all C files matching the specified file glob.
-func (c *Compiler) CompileC() error {
- files, _ := filepath.Glob(c.srcDir + "/*.c")
-
- log.Infof("Compiling C if outdated (%s/*.c) %s", c.srcDir,
- strings.Join(files, " "))
+func compilerTypeToExts(compilerType int) ([]string, error) {
+ switch compilerType {
+ case COMPILER_TYPE_C:
+ return []string{"c"}, nil
+ case COMPILER_TYPE_ASM:
+ return []string{"s", "S"}, nil
+ case COMPILER_TYPE_CPP:
+ return []string{"cc", "cpp", "cxx"}, nil
+ case COMPILER_TYPE_ARCHIVE:
+ return []string{"a"}, nil
+ default:
+ return nil, util.NewNewtError("Wrong compiler type specified to " +
+ "compilerTypeToExts")
+ }
+}
- for _, file := range files {
- file = filepath.ToSlash(file)
+// Compiles all C files matching the specified file glob.
+func (c *Compiler) CompileC(filename string) error {
+ filename = filepath.ToSlash(filename)
- if shouldIgnore := c.shouldIgnoreFile(file); shouldIgnore {
- log.Infof("Ignoring %s because package dictates it.", file)
- continue
- }
+ if c.shouldIgnoreFile(filename) {
+ log.Infof("Ignoring %s because package dictates it.", filename)
+ return nil
+ }
- compileRequired, err := c.depTracker.CompileRequired(file,
- COMPILER_TYPE_C)
- if err != nil {
- return err
- }
- if compileRequired {
- err = c.CompileFile(file, COMPILER_TYPE_C)
- } else {
- err = c.SkipSourceFile(file)
- }
- if err != nil {
- return err
- }
+ compileRequired, err := c.depTracker.CompileRequired(filename,
+ COMPILER_TYPE_C)
+ if err != nil {
+ return err
+ }
+ if compileRequired {
+ err = c.CompileFile(filename, COMPILER_TYPE_C)
+ } else {
+ err = c.SkipSourceFile(filename)
+ }
+ if err != nil {
+ return err
}
return nil
}
// Compiles all CPP files
-func (c *Compiler) CompileCpp() error {
- files, _ := filepath.Glob(c.srcDir + "/*.cc")
- moreFiles, _ := filepath.Glob(c.srcDir + "/*.cpp")
- files = append(files, moreFiles...)
- moreFiles, _ = filepath.Glob(c.srcDir + "/*.cxx")
- files = append(files, moreFiles...)
-
- wd, err := os.Getwd()
+func (c *Compiler) CompileCpp(filename string) error {
+ filename = filepath.ToSlash(filename)
+
+ if c.shouldIgnoreFile(filename) {
+ log.Infof("Ignoring %s because package dictates it.", filename)
+ return nil
+ }
+
+ compileRequired, err := c.depTracker.CompileRequired(filename,
+ COMPILER_TYPE_CPP)
if err != nil {
return err
}
- wd = filepath.ToSlash(filepath.Clean(wd))
- log.Infof("Working in dir (%s)", wd)
-
- log.Infof("Compiling CC if outdated (%s/*.cc) %s", wd,
- strings.Join(files, " "))
-
- for _, file := range files {
- file = filepath.ToSlash(file)
-
- if shouldIgnore := c.shouldIgnoreFile(file); shouldIgnore {
- log.Infof("Ignoring %s because package dictates it.", file)
- }
- compileRequired, err := c.depTracker.CompileRequired(file,
- COMPILER_TYPE_CPP)
- if err != nil {
- return err
- }
-
- if compileRequired {
- err = c.CompileFile(file, COMPILER_TYPE_CPP)
- } else {
- err = c.SkipSourceFile(file)
- }
+ if compileRequired {
+ err = c.CompileFile(filename, COMPILER_TYPE_CPP)
+ } else {
+ err = c.SkipSourceFile(filename)
+ }
- if err != nil {
- return err
- }
+ if err != nil {
+ return err
}
return nil
@@ -610,41 +619,26 @@ func (c *Compiler) CompileCpp() error {
//
// @param match The file glob specifying which assembly files
// to compile.
-func (c *Compiler) CompileAs() error {
- files, _ := filepath.Glob(c.srcDir + "/*.s")
- moreFiles, _ := filepath.Glob(c.srcDir + "/*.S")
- files = append(files, moreFiles...)
+func (c *Compiler) CompileAs(filename string) error {
+ filename = filepath.ToSlash(filename)
+
+ if c.shouldIgnoreFile(filename) {
+ log.Infof("Ignoring %s because package dictates it.", filename)
+ return nil
+ }
- wd, err := os.Getwd()
+ compileRequired, err := c.depTracker.CompileRequired(filename,
+ COMPILER_TYPE_ASM)
if err != nil {
return err
}
- wd = filepath.ToSlash(filepath.Clean(wd))
- log.Infof("Working in dir (%s)", wd)
-
- log.Infof("Compiling assembly if outdated (%s/*.s) %s", wd,
- strings.Join(files, " "))
- for _, file := range files {
- file = filepath.ToSlash(file)
-
- if shouldIgnore := c.shouldIgnoreFile(file); shouldIgnore {
- log.Infof("Ignoring %s because package dictates it.", file)
- continue
- }
-
- compileRequired, err := c.depTracker.CompileRequired(file,
- COMPILER_TYPE_ASM)
- if err != nil {
- return err
- }
- if compileRequired {
- err = c.CompileFile(file, COMPILER_TYPE_ASM)
- } else {
- err = c.SkipSourceFile(file)
- }
- if err != nil {
- return err
- }
+ if compileRequired {
+ err = c.CompileFile(filename, COMPILER_TYPE_ASM)
+ } else {
+ err = c.SkipSourceFile(filename)
+ }
+ if err != nil {
+ return err
}
return nil
@@ -654,52 +648,46 @@ func (c *Compiler) CompileAs() error {
//
// @param match The file glob specifying which assembly files
// to compile.
-func (c *Compiler) CopyArchive() error {
- files, _ := filepath.Glob(c.srcDir + "/*.a")
-
- log.Infof("Copying archive if outdated (%s/*.a) %s", c.srcDir,
- strings.Join(files, " "))
- for _, file := range files {
- file = filepath.ToSlash(file)
+func (c *Compiler) CopyArchive(filename string) error {
+ filename = filepath.ToSlash(filename)
- if shouldIgnore := c.shouldIgnoreFile(file); shouldIgnore {
- log.Infof("Ignoring %s because package dictates it.", file)
- continue
- }
+ if c.shouldIgnoreFile(filename) {
+ log.Infof("Ignoring %s because package dictates it.", filename)
+ return nil
+ }
- tgtFile := c.dstFilePath(file) + ".a"
- copyRequired, err := c.depTracker.CopyRequired(file)
- if err != nil {
- return err
- }
- if copyRequired {
- err = util.CopyFile(file, tgtFile)
- util.StatusMessage(util.VERBOSITY_DEFAULT, "copying %s\n",
- filepath.ToSlash(tgtFile))
- }
+ tgtFile := c.dstFilePath(filename) + ".a"
+ copyRequired, err := c.depTracker.CopyRequired(filename)
+ if err != nil {
+ return err
+ }
+ if copyRequired {
+ err = util.CopyFile(filename, tgtFile)
+ util.StatusMessage(util.VERBOSITY_DEFAULT, "copying %s\n",
+ filepath.ToSlash(tgtFile))
+ }
- if err != nil {
- return err
- }
+ if err != nil {
+ return err
}
return nil
}
func (c *Compiler) processEntry(node os.FileInfo, cType int,
- ignDirs []string) error {
+ ignDirs []string) ([]CompilerJob, error) {
// check to see if we ignore this element
- for _, entry := range ignDirs {
- if entry == node.Name() {
- return nil
+ for _, dir := range ignDirs {
+ if dir == node.Name() {
+ return nil, nil
}
}
// Check in the user specified ignore directories
- for _, entry := range c.info.IgnoreDirs {
- if entry.MatchString(node.Name()) {
- return nil
+ for _, dir := range c.info.IgnoreDirs {
+ if dir.MatchString(node.Name()) {
+ return nil, nil
}
}
@@ -711,61 +699,88 @@ func (c *Compiler) processEntry(node os.FileInfo, cType int,
c.srcDir += "/" + node.Name()
c.dstDir += "/" + node.Name()
- err := c.RecursiveCompile(cType, ignDirs)
+ entries, err := c.RecursiveCollectEntries(cType, ignDirs)
- // Restore the compiler destination directory now that the child directory
- // has been fully built.
+ // Restore the compiler destination directory now that the child
+ // directory has been fully built.
c.srcDir = prevSrcDir
c.dstDir = prevDstDir
- return err
+ return entries, err
}
-func (c *Compiler) RecursiveCompile(cType int, ignDirs []string) error {
+func (c *Compiler) RecursiveCollectEntries(cType int,
+ ignDirs []string) ([]CompilerJob, error) {
+
// Make sure the compiler package info is added to the global set.
c.ensureLclInfoAdded()
if err := os.Chdir(c.baseDir); err != nil {
- return util.ChildNewtError(err)
+ return nil, util.ChildNewtError(err)
}
// Get a list of files in the current directory, and if they are a
// directory, and that directory is not in the ignDirs variable, then
// recurse into that directory and compile the files in there
- dirList, err := ioutil.ReadDir(c.srcDir)
+ ls, err := ioutil.ReadDir(c.srcDir)
if err != nil {
- return util.NewNewtError(err.Error())
+ return nil, util.NewNewtError(err.Error())
}
- for _, node := range dirList {
+ entries := []CompilerJob{}
+ for _, node := range ls {
if node.IsDir() {
- err = c.processEntry(node, cType, ignDirs)
+ subEntries, err := c.processEntry(node, cType, ignDirs)
if err != nil {
- return err
+ return nil, err
}
+
+ entries = append(entries, subEntries...)
}
}
- switch cType {
+ exts, err := compilerTypeToExts(cType)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, ext := range exts {
+ files, _ := filepath.Glob(c.srcDir + "/*." + ext)
+ for _, file := range files {
+ entries = append(entries, CompilerJob{
+ Filename: file,
+ Compiler: c,
+ CompilerType: cType,
+ })
+ }
+ }
+
+ return entries, nil
+}
+
+func RunJob(record CompilerJob) error {
+ switch record.CompilerType {
case COMPILER_TYPE_C:
- return c.CompileC()
+ return record.Compiler.CompileC(record.Filename)
case COMPILER_TYPE_ASM:
- return c.CompileAs()
+ return record.Compiler.CompileAs(record.Filename)
case COMPILER_TYPE_CPP:
- return c.CompileCpp()
+ return record.Compiler.CompileCpp(record.Filename)
case COMPILER_TYPE_ARCHIVE:
- return c.CopyArchive()
+ return record.Compiler.CopyArchive(record.Filename)
default:
return util.NewNewtError("Wrong compiler type specified to " +
- "RecursiveCompile")
+ "RunJob")
}
}
func (c *Compiler) getObjFiles(baseObjFiles []string) []string {
- for objName, _ := range c.ObjPathList {
+ c.mutex.Lock()
+ for objName, _ := range c.objPathList {
baseObjFiles = append(baseObjFiles, objName)
}
+ c.mutex.Unlock()
sort.Strings(baseObjFiles)
return baseObjFiles