You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mynewt.apache.org by cc...@apache.org on 2016/11/10 19:55:10 UTC
[1/3] incubator-mynewt-newt git commit: newt - Get repository.yml
from priv github repos.
Repository: incubator-mynewt-newt
Updated Branches:
refs/heads/develop 93cf30fbe -> 00ec2a349
newt - Get repository.yml from priv github repos.
To access a private repository, newt needs to be configured with either:
* Access token for the repository
or,
* Basic auth login and password for the user
(To create a github access token, see:
https://help.github.com/articles/creating-an-access-token-for-command-line-use/).
There are two ways to specify this information, as shown below. In
these examples, I specify both a token and a login/password, but you
only need to specify one of these.
1. project.yml (probably world-readable and therefore not secure):
repository.my-private-repo:
type: github
vers: 0-dev
user: owner-of-repo
repo: repo-name
token: '8ab6433f8971b05c2a9c3341533e8ddb754e404e'
login: githublogin
password: githubpassword
(The last three lines are the new settings)
2. $HOME/.newt/repos.yml
repository.my-private-repo:
token: '8ab6433f8971b05c2a9c3341533e8ddb754e404e'
login: githublogin
password: githubpassword
If both a token and a login+password are specified, newt uses the token.
If both the project.yml file and the private repos.yml file specify
security credentials, newt uses the project.yml settings.
There is one annoyance: when newt downloads the actual repo content, as
opposed to just the repository.yml file, it doesn't use the same
mechanism. Instead, it invokes the git command line tool. This is an
annoyance because the user cannot use the same access token for all git
operations. This is something that we should fix in the future.
Project: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/commit/c6733650
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/tree/c6733650
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/diff/c6733650
Branch: refs/heads/develop
Commit: c673365078887d28f313fe7fe053a5509c89cd87
Parents: f0a27b6
Author: Christopher Collins <cc...@apache.org>
Authored: Mon Sep 26 18:51:39 2016 -0700
Committer: Christopher Collins <cc...@apache.org>
Committed: Mon Sep 26 19:02:22 2016 -0700
----------------------------------------------------------------------
newt/downloader/downloader.go | 45 ++++++++++++++++++---
newt/newtutil/newtutil.go | 81 ++++++++++++++++++++++++++++++++++++++
newt/project/project.go | 24 +++++++++++
newt/repo/repo.go | 4 +-
4 files changed, 146 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c6733650/newt/downloader/downloader.go
----------------------------------------------------------------------
diff --git a/newt/downloader/downloader.go b/newt/downloader/downloader.go
index 1a816d5..d356c10 100644
--- a/newt/downloader/downloader.go
+++ b/newt/downloader/downloader.go
@@ -48,6 +48,13 @@ type GithubDownloader struct {
GenericDownloader
User string
Repo string
+
+ // Github access token for private repositories.
+ Token string
+
+ // Basic authentication login and password for private repositories.
+ Login string
+ Password string
}
func (gd *GenericDownloader) Branch() string {
@@ -64,17 +71,43 @@ func (gd *GenericDownloader) TempDir() (string, error) {
}
func (gd *GithubDownloader) FetchFile(name string, dest string) error {
- fmtStr := "https://raw.githubusercontent.com/%s/%s/%s/%s"
- url := fmt.Sprintf(fmtStr, gd.User, gd.Repo, gd.Branch(), name)
+ url := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s?ref=%s",
+ gd.User, gd.Repo, name, gd.Branch())
+
+ req, err := http.NewRequest("GET", url, nil)
+ req.Header.Add("Accept", "application/vnd.github.v3.raw")
+
+ if gd.Token != "" {
+ // XXX: Add command line option to include token in log.
+ log.Debugf("Using authorization token")
+ req.Header.Add("Authorization", "token "+gd.Token)
+ } else if gd.Login != "" && gd.Password != "" {
+ // XXX: Add command line option to include password in log.
+ log.Debugf("Using basic auth; login=%s", gd.Login)
+ req.SetBasicAuth(gd.Login, gd.Password)
+ }
log.Debugf("Fetching file %s (url: %s) to %s", name, url, dest)
-
- rsp, err := http.Get(url)
+ client := &http.Client{}
+ rsp, err := client.Do(req)
if err != nil {
return util.NewNewtError(err.Error())
}
defer rsp.Body.Close()
+ if rsp.StatusCode != http.StatusOK {
+ errMsg := fmt.Sprintf("Failed to download '%s'; status=%s",
+ url, rsp.Status)
+ switch rsp.StatusCode {
+ case http.StatusNotFound:
+ errMsg += "; URL incorrect or repository private?"
+ case http.StatusUnauthorized:
+ errMsg += "; credentials incorrect?"
+ }
+
+ return util.NewNewtError(errMsg)
+ }
+
handle, err := os.Create(dest)
if err != nil {
return util.NewNewtError(err.Error())
@@ -104,9 +137,9 @@ func (gd *GithubDownloader) DownloadRepo(commit string) (string, error) {
branch := "master"
url := fmt.Sprintf("https://github.com/%s/%s.git", gd.User, gd.Repo)
- util.StatusMessage(util.VERBOSITY_VERBOSE, fmt.Sprintf("Downloading "+
+ util.StatusMessage(util.VERBOSITY_VERBOSE, "Downloading "+
"repository %s (branch: %s; commit: %s) at %s\n", gd.Repo, branch,
- commit, url))
+ commit, url)
gitPath, err := exec.LookPath("git")
if err != nil {
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c6733650/newt/newtutil/newtutil.go
----------------------------------------------------------------------
diff --git a/newt/newtutil/newtutil.go b/newt/newtutil/newtutil.go
index 0daa992..6e071f6 100644
--- a/newt/newtutil/newtutil.go
+++ b/newt/newtutil/newtutil.go
@@ -21,10 +21,14 @@ package newtutil
import (
"fmt"
+ "os/user"
"sort"
"strconv"
"strings"
+ log "github.com/Sirupsen/logrus"
+ "github.com/spf13/cast"
+
"mynewt.apache.org/newt/util"
"mynewt.apache.org/newt/viper"
)
@@ -32,6 +36,83 @@ import (
var NewtVersionStr string = "Apache Newt (incubating) version: 0.9.0"
var NewtBlinkyTag string = "mynewt_0_9_0_tag"
+const NEWTRC_DIR string = ".newt"
+const REPOS_FILENAME string = "repos.yml"
+
+// Contains general newt settings read from $HOME/.newt
+var newtrc *viper.Viper
+
+func readNewtrc() *viper.Viper {
+ usr, err := user.Current()
+ if err != nil {
+ log.Warn("Failed to obtain user name")
+ return viper.New()
+ }
+
+ dir := usr.HomeDir + "/" + NEWTRC_DIR
+ v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
+ if err != nil {
+ log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
+ return viper.New()
+ }
+
+ return v
+}
+
+func Newtrc() *viper.Viper {
+ if newtrc != nil {
+ return newtrc
+ }
+
+ newtrc = readNewtrc()
+ return newtrc
+}
+
+func GetSliceFeatures(v *viper.Viper, features map[string]bool,
+ key string) []interface{} {
+
+ val := v.Get(key)
+ vals := []interface{}{val}
+
+ // Process the features in alphabetical order to ensure consistent
+ // results across repeated runs.
+ var featureKeys []string
+ for feature, _ := range features {
+ featureKeys = append(featureKeys, feature)
+ }
+ sort.Strings(featureKeys)
+
+ for _, feature := range featureKeys {
+ overwriteVal := v.Get(key + "." + feature + ".OVERWRITE")
+ if overwriteVal != nil {
+ return []interface{}{overwriteVal}
+ }
+
+ appendVal := v.Get(key + "." + feature)
+ if appendVal != nil {
+ vals = append(vals, appendVal)
+ }
+ }
+
+ return vals
+}
+
+func GetStringMapFeatures(v *viper.Viper, features map[string]bool,
+ key string) map[string]interface{} {
+
+ result := map[string]interface{}{}
+
+ slice := GetSliceFeatures(v, features, key)
+ for _, itf := range slice {
+ sub := cast.ToStringMap(itf)
+ for k, v := range sub {
+ result[k] = v
+ }
+ }
+
+ return result
+}
+
func GetStringFeatures(v *viper.Viper, features map[string]bool,
key string) string {
val := v.GetString(key)
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c6733650/newt/project/project.go
----------------------------------------------------------------------
diff --git a/newt/project/project.go b/newt/project/project.go
index 0673365..3703f1c 100644
--- a/newt/project/project.go
+++ b/newt/project/project.go
@@ -30,6 +30,7 @@ import (
"mynewt.apache.org/newt/newt/downloader"
"mynewt.apache.org/newt/newt/interfaces"
+ "mynewt.apache.org/newt/newt/newtutil"
"mynewt.apache.org/newt/newt/pkg"
"mynewt.apache.org/newt/newt/repo"
"mynewt.apache.org/newt/util"
@@ -385,6 +386,29 @@ func (proj *Project) loadRepo(rname string, v *viper.Viper) error {
dl.User = repoVars["user"]
dl.Repo = repoVars["repo"]
+ // The project.yml file can contain github access tokens and authentication
+ // credentials, but this file is probably world-readable and therefore not
+ // a great place for this.
+ dl.Token = repoVars["token"]
+ dl.Login = repoVars["login"]
+ dl.Password = repoVars["password"]
+
+ // Alternatively, the user can put security material in
+ // $HOME/.newt/repos.yml.
+ newtrc := newtutil.Newtrc()
+ privRepo := newtrc.GetStringMapString("repository." + rname)
+ if privRepo != nil {
+ if dl.Token == "" {
+ dl.Token = privRepo["token"]
+ }
+ if dl.Login == "" {
+ dl.Login = privRepo["login"]
+ }
+ if dl.Password == "" {
+ dl.Password = privRepo["password"]
+ }
+ }
+
r, err := repo.NewRepo(rname, rversreq, dl)
if err != nil {
return err
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/c6733650/newt/repo/repo.go
----------------------------------------------------------------------
diff --git a/newt/repo/repo.go b/newt/repo/repo.go
index 9b3fada..62bb12b 100644
--- a/newt/repo/repo.go
+++ b/newt/repo/repo.go
@@ -340,8 +340,8 @@ func (r *Repo) UpdateDesc() ([]*Repo, bool, error) {
func (r *Repo) DownloadDesc() error {
dl := r.downloader
- util.StatusMessage(util.VERBOSITY_VERBOSE, fmt.Sprintf("Downloading "+
- "repository description for %s...", r.Name()))
+ util.StatusMessage(util.VERBOSITY_VERBOSE, "Downloading "+
+ "repository description for %s...\n", r.Name())
// Configuration path
cpath := r.repoFilePath()
[2/3] incubator-mynewt-newt git commit: Merge remote-tracking branch
'apache/master' into develop
Posted by cc...@apache.org.
Merge remote-tracking branch 'apache/master' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/commit/48599c13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/tree/48599c13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/diff/48599c13
Branch: refs/heads/develop
Commit: 48599c13235dfe62b6928849afd8d58a497cd900
Parents: 93cf30f c673365
Author: Christopher Collins <cc...@apache.org>
Authored: Thu Nov 10 11:38:28 2016 -0800
Committer: Christopher Collins <cc...@apache.org>
Committed: Thu Nov 10 11:38:28 2016 -0800
----------------------------------------------------------------------
newt/newtutil/newtutil.go | 77 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 77 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/48599c13/newt/newtutil/newtutil.go
----------------------------------------------------------------------
diff --cc newt/newtutil/newtutil.go
index d7369d9,6e071f6..79b3cef
--- a/newt/newtutil/newtutil.go
+++ b/newt/newtutil/newtutil.go
@@@ -33,52 -33,52 +33,129 @@@ import
"mynewt.apache.org/newt/viper"
)
-var NewtVersionStr string = "Apache Newt (incubating) version: 0.9.0"
-var NewtBlinkyTag string = "mynewt_0_9_0_tag"
+var NewtVersionStr string = "Apache Newt (incubating) version: 1.0.0-develop"
+var NewtBlinkyTag string = "develop"
+
+const NEWTRC_DIR string = ".newt"
+const REPOS_FILENAME string = "repos.yml"
+
+// Contains general newt settings read from $HOME/.newt
+var newtrc *viper.Viper
+
+func readNewtrc() *viper.Viper {
+ usr, err := user.Current()
+ if err != nil {
+ log.Warn("Failed to obtain user name")
+ return viper.New()
+ }
+
+ dir := usr.HomeDir + "/" + NEWTRC_DIR
+ v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
+ if err != nil {
+ log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
+ return viper.New()
+ }
+
+ return v
+}
+
+func Newtrc() *viper.Viper {
+ if newtrc != nil {
+ return newtrc
+ }
+
+ newtrc = readNewtrc()
+ return newtrc
+}
+
+func GetSliceFeatures(v *viper.Viper, features map[string]bool,
+ key string) []interface{} {
+
+ val := v.Get(key)
+ vals := []interface{}{val}
+
+ // Process the features in alphabetical order to ensure consistent
+ // results across repeated runs.
+ var featureKeys []string
+ for feature, _ := range features {
+ featureKeys = append(featureKeys, feature)
++ }
++ sort.Strings(featureKeys)
++
++ for _, feature := range featureKeys {
++ overwriteVal := v.Get(key + "." + feature + ".OVERWRITE")
++ if overwriteVal != nil {
++ return []interface{}{overwriteVal}
++ }
++
++ appendVal := v.Get(key + "." + feature)
++ if appendVal != nil {
++ vals = append(vals, appendVal)
++ }
++ }
++
++ return vals
++}
++
++func GetStringMapFeatures(v *viper.Viper, features map[string]bool,
++ key string) map[string]interface{} {
++
++ result := map[string]interface{}{}
++
++ slice := GetSliceFeatures(v, features, key)
++ for _, itf := range slice {
++ sub := cast.ToStringMap(itf)
++ for k, v := range sub {
++ result[k] = v
++ }
++ }
++
++ return result
++}
+
+ const NEWTRC_DIR string = ".newt"
+ const REPOS_FILENAME string = "repos.yml"
+
+ // Contains general newt settings read from $HOME/.newt
+ var newtrc *viper.Viper
+
+ func readNewtrc() *viper.Viper {
+ usr, err := user.Current()
+ if err != nil {
+ log.Warn("Failed to obtain user name")
+ return viper.New()
+ }
+
+ dir := usr.HomeDir + "/" + NEWTRC_DIR
+ v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
+ if err != nil {
+ log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
+ return viper.New()
+ }
+
+ return v
+ }
+
+ func Newtrc() *viper.Viper {
+ if newtrc != nil {
+ return newtrc
+ }
+
+ newtrc = readNewtrc()
+ return newtrc
+ }
+
+ func GetSliceFeatures(v *viper.Viper, features map[string]bool,
+ key string) []interface{} {
+
+ val := v.Get(key)
+ vals := []interface{}{val}
+
+ // Process the features in alphabetical order to ensure consistent
+ // results across repeated runs.
+ var featureKeys []string
+ for feature, _ := range features {
+ featureKeys = append(featureKeys, feature)
}
sort.Strings(featureKeys)
[3/3] incubator-mynewt-newt git commit: newt - Remove duplicate code
introduced in merge.
Posted by cc...@apache.org.
newt - Remove duplicate code introduced in merge.
Project: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/commit/00ec2a34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/tree/00ec2a34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/diff/00ec2a34
Branch: refs/heads/develop
Commit: 00ec2a3494d2dc0d64e0f347bae6ba5f184f16ed
Parents: 48599c1
Author: Christopher Collins <cc...@apache.org>
Authored: Thu Nov 10 11:54:29 2016 -0800
Committer: Christopher Collins <cc...@apache.org>
Committed: Thu Nov 10 11:54:29 2016 -0800
----------------------------------------------------------------------
newt/newtutil/newtutil.go | 77 ------------------------------------------
1 file changed, 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/00ec2a34/newt/newtutil/newtutil.go
----------------------------------------------------------------------
diff --git a/newt/newtutil/newtutil.go b/newt/newtutil/newtutil.go
index 79b3cef..d7369d9 100644
--- a/newt/newtutil/newtutil.go
+++ b/newt/newtutil/newtutil.go
@@ -113,83 +113,6 @@ func GetStringMapFeatures(v *viper.Viper, features map[string]bool,
return result
}
-const NEWTRC_DIR string = ".newt"
-const REPOS_FILENAME string = "repos.yml"
-
-// Contains general newt settings read from $HOME/.newt
-var newtrc *viper.Viper
-
-func readNewtrc() *viper.Viper {
- usr, err := user.Current()
- if err != nil {
- log.Warn("Failed to obtain user name")
- return viper.New()
- }
-
- dir := usr.HomeDir + "/" + NEWTRC_DIR
- v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
- if err != nil {
- log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
- return viper.New()
- }
-
- return v
-}
-
-func Newtrc() *viper.Viper {
- if newtrc != nil {
- return newtrc
- }
-
- newtrc = readNewtrc()
- return newtrc
-}
-
-func GetSliceFeatures(v *viper.Viper, features map[string]bool,
- key string) []interface{} {
-
- val := v.Get(key)
- vals := []interface{}{val}
-
- // Process the features in alphabetical order to ensure consistent
- // results across repeated runs.
- var featureKeys []string
- for feature, _ := range features {
- featureKeys = append(featureKeys, feature)
- }
- sort.Strings(featureKeys)
-
- for _, feature := range featureKeys {
- overwriteVal := v.Get(key + "." + feature + ".OVERWRITE")
- if overwriteVal != nil {
- return []interface{}{overwriteVal}
- }
-
- appendVal := v.Get(key + "." + feature)
- if appendVal != nil {
- vals = append(vals, appendVal)
- }
- }
-
- return vals
-}
-
-func GetStringMapFeatures(v *viper.Viper, features map[string]bool,
- key string) map[string]interface{} {
-
- result := map[string]interface{}{}
-
- slice := GetSliceFeatures(v, features, key)
- for _, itf := range slice {
- sub := cast.ToStringMap(itf)
- for k, v := range sub {
- result[k] = v
- }
- }
-
- return result
-}
-
func GetStringFeatures(v *viper.Viper, features map[string]bool,
key string) string {
val := v.GetString(key)