You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mynewt.apache.org by cc...@apache.org on 2016/09/27 01:59:13 UTC
incubator-mynewt-newt git commit: newt - Get repository.yml from priv
github repos.
Repository: incubator-mynewt-newt
Updated Branches:
refs/heads/develop 498e1d703 -> 9f6f9d9e8
newt - Get repository.yml from priv github repos.
To access a private repository, newt needs to be configured with either:
* Access token for the repository
or,
* Basic auth login and password for the user
(To create a github access token, see:
https://help.github.com/articles/creating-an-access-token-for-command-line-use/).
There are two ways to specify this information, as shown below. In
these examples, I specify both a token and a login/password, but you
only need to specify one of these.
1. project.yml (probably world-readable and therefore not secure):
repository.my-private-repo:
type: github
vers: 0-dev
user: owner-of-repo
repo: repo-name
token: '8ab6433f8971b05c2a9c3341533e8ddb754e404e'
login: githublogin
password: githubpassword
(The last three lines are the new settings)
2. $HOME/.newt/repos.yml
repository.my-private-repo:
token: '8ab6433f8971b05c2a9c3341533e8ddb754e404e'
login: githublogin
password: githubpassword
If both a token and a login+password are specified, newt uses the token.
If both the project.yml file and the private repos.yml file specify
security credentials, newt uses the project.yml settings.
There is one annoyance: when newt downloads the actual repo content, as
opposed to just the repository.yml file, it doesn't use the same
mechanism. Instead, it invokes the git command line tool. This is an
annoyance because the user cannot use the same access token for all git
operations. This is something that we should fix in the future.
Project: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/commit/9f6f9d9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/tree/9f6f9d9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/diff/9f6f9d9e
Branch: refs/heads/develop
Commit: 9f6f9d9e8eb4aa10b3d58be4de43cdf8ecb2ae62
Parents: 498e1d7
Author: Christopher Collins <cc...@apache.org>
Authored: Mon Sep 26 18:51:39 2016 -0700
Committer: Christopher Collins <cc...@apache.org>
Committed: Mon Sep 26 18:51:39 2016 -0700
----------------------------------------------------------------------
newt/downloader/downloader.go | 48 ++++++++++++++++++++++++++++++--------
newt/newtutil/newtutil.go | 34 +++++++++++++++++++++++++++
newt/project/project.go | 23 ++++++++++++++++++
newt/repo/repo.go | 4 ++--
4 files changed, 97 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/9f6f9d9e/newt/downloader/downloader.go
----------------------------------------------------------------------
diff --git a/newt/downloader/downloader.go b/newt/downloader/downloader.go
index df18968..7cf1892 100644
--- a/newt/downloader/downloader.go
+++ b/newt/downloader/downloader.go
@@ -48,6 +48,13 @@ type GithubDownloader struct {
GenericDownloader
User string
Repo string
+
+ // Github access token for private repositories.
+ Token string
+
+ // Basic authentication login and password for private repositories.
+ Login string
+ Password string
}
func (gd *GenericDownloader) Branch() string {
@@ -64,21 +71,42 @@ func (gd *GenericDownloader) TempDir() (string, error) {
}
func (gd *GithubDownloader) FetchFile(name string, dest string) error {
- fmtStr := "https://raw.githubusercontent.com/%s/%s/%s/%s"
- url := fmt.Sprintf(fmtStr, gd.User, gd.Repo, gd.Branch(), name)
+ url := fmt.Sprintf("https://api.github.com/repos/%s/%s/contents/%s?ref=%s",
+ gd.User, gd.Repo, name, gd.Branch())
+
+ req, err := http.NewRequest("GET", url, nil)
+ req.Header.Add("Accept", "application/vnd.github.v3.raw")
+
+ if gd.Token != "" {
+ // XXX: Add command line option to include token in log.
+ log.Debugf("Using authorization token")
+ req.Header.Add("Authorization", "token "+gd.Token)
+ } else if gd.Login != "" && gd.Password != "" {
+ // XXX: Add command line option to include password in log.
+ log.Debugf("Using basic auth; login=%s", gd.Login)
+ req.SetBasicAuth(gd.Login, gd.Password)
+ }
log.Debugf("Fetching file %s (url: %s) to %s", name, url, dest)
-
- rsp, err := http.Get(url)
+ client := &http.Client{}
+ rsp, err := client.Do(req)
if err != nil {
return util.NewNewtError(err.Error())
}
+ defer rsp.Body.Close()
+
if rsp.StatusCode != http.StatusOK {
- return util.NewNewtError(fmt.Sprintf(
- "Failed to download '%s' from https://github.com/%s/%s: %s\n",
- name, gd.User, gd.Repo, rsp.Status))
+ errMsg := fmt.Sprintf("Failed to download '%s'; status=%s",
+ url, rsp.Status)
+ switch rsp.StatusCode {
+ case http.StatusNotFound:
+ errMsg += "; URL incorrect or repository private?"
+ case http.StatusUnauthorized:
+ errMsg += "; credentials incorrect?"
+ }
+
+ return util.NewNewtError(errMsg)
}
- defer rsp.Body.Close()
handle, err := os.Create(dest)
if err != nil {
@@ -109,9 +137,9 @@ func (gd *GithubDownloader) DownloadRepo(commit string) (string, error) {
branch := "master"
url := fmt.Sprintf("https://github.com/%s/%s.git", gd.User, gd.Repo)
- util.StatusMessage(util.VERBOSITY_VERBOSE, fmt.Sprintf("Downloading "+
+ util.StatusMessage(util.VERBOSITY_VERBOSE, "Downloading "+
"repository %s (branch: %s; commit: %s) at %s\n", gd.Repo, branch,
- commit, url))
+ commit, url)
gitPath, err := exec.LookPath("git")
if err != nil {
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/9f6f9d9e/newt/newtutil/newtutil.go
----------------------------------------------------------------------
diff --git a/newt/newtutil/newtutil.go b/newt/newtutil/newtutil.go
index 947ab49..0056201 100644
--- a/newt/newtutil/newtutil.go
+++ b/newt/newtutil/newtutil.go
@@ -23,10 +23,12 @@ import (
"fmt"
"io"
"os"
+ "os/user"
"sort"
"strconv"
"strings"
+ log "github.com/Sirupsen/logrus"
"github.com/spf13/cast"
"mynewt.apache.org/newt/util"
@@ -36,6 +38,38 @@ import (
var NewtVersionStr string = "Apache Newt (incubating) version: 0.9.0"
var NewtBlinkyTag string = "mynewt_0_9_0_tag"
+const NEWTRC_DIR string = ".newt"
+const REPOS_FILENAME string = "repos.yml"
+
+// Contains general newt settings read from $HOME/.newt
+var newtrc *viper.Viper
+
+func readNewtrc() *viper.Viper {
+ usr, err := user.Current()
+ if err != nil {
+ log.Warn("Failed to obtain user name")
+ return viper.New()
+ }
+
+ dir := usr.HomeDir + "/" + NEWTRC_DIR
+ v, err := util.ReadConfig(dir, strings.TrimSuffix(REPOS_FILENAME, ".yml"))
+ if err != nil {
+ log.Debugf("Failed to read %s/%s file", dir, REPOS_FILENAME)
+ return viper.New()
+ }
+
+ return v
+}
+
+func Newtrc() *viper.Viper {
+ if newtrc != nil {
+ return newtrc
+ }
+
+ newtrc = readNewtrc()
+ return newtrc
+}
+
func GetSliceFeatures(v *viper.Viper, features map[string]bool,
key string) []interface{} {
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/9f6f9d9e/newt/project/project.go
----------------------------------------------------------------------
diff --git a/newt/project/project.go b/newt/project/project.go
index 7daa694..5099e6f 100644
--- a/newt/project/project.go
+++ b/newt/project/project.go
@@ -381,6 +381,29 @@ func (proj *Project) loadRepo(rname string, v *viper.Viper) error {
dl.User = repoVars["user"]
dl.Repo = repoVars["repo"]
+ // The project.yml file can contain github access tokens and authentication
+ // credentials, but this file is probably world-readable and therefore not
+ // a great place for this.
+ dl.Token = repoVars["token"]
+ dl.Login = repoVars["login"]
+ dl.Password = repoVars["password"]
+
+ // Alternatively, the user can put security material in
+ // $HOME/.newt/repos.yml.
+ newtrc := newtutil.Newtrc()
+ privRepo := newtrc.GetStringMapString("repository." + rname)
+ if privRepo != nil {
+ if dl.Token == "" {
+ dl.Token = privRepo["token"]
+ }
+ if dl.Login == "" {
+ dl.Login = privRepo["login"]
+ }
+ if dl.Password == "" {
+ dl.Password = privRepo["password"]
+ }
+ }
+
r, err := repo.NewRepo(rname, rversreq, dl)
if err != nil {
return err
http://git-wip-us.apache.org/repos/asf/incubator-mynewt-newt/blob/9f6f9d9e/newt/repo/repo.go
----------------------------------------------------------------------
diff --git a/newt/repo/repo.go b/newt/repo/repo.go
index 8ac6a0e..a7b0253 100644
--- a/newt/repo/repo.go
+++ b/newt/repo/repo.go
@@ -428,8 +428,8 @@ func (r *Repo) UpdateDesc() ([]*Repo, bool, error) {
func (r *Repo) DownloadDesc() error {
dl := r.downloader
- util.StatusMessage(util.VERBOSITY_VERBOSE, fmt.Sprintf("Downloading "+
- "repository description for %s...", r.Name()))
+ util.StatusMessage(util.VERBOSITY_VERBOSE, "Downloading "+
+ "repository description for %s...\n", r.Name())
// Configuration path
cpath := r.repoFilePath()