You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@camel.apache.org by lb...@apache.org on 2020/10/13 07:48:13 UTC
[camel-k] branch master updated: Add suport for gists #1740
This is an automated email from the ASF dual-hosted git repository.
lburgazzoli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/camel-k.git
The following commit(s) were added to refs/heads/master by this push:
new b3ae436 Add suport for gists #1740
b3ae436 is described below
commit b3ae43691dea0afa6eb05a9a46c9d486f13656e4
Author: Luca Burgazzoli <lb...@gmail.com>
AuthorDate: Mon Oct 5 15:53:54 2020 +0200
Add suport for gists #1740
---
.../ROOT/pages/running/run-from-github.adoc | 57 +++++-
e2e/common/run_test.go | 64 +++++--
go.mod | 2 +
go.sum | 4 +
pkg/cmd/modeline.go | 106 +++++------
pkg/cmd/modeline_test.go | 8 +-
pkg/cmd/run.go | 96 ++--------
pkg/cmd/util.go | 13 ++
pkg/cmd/{util_getter.go => util_content.go} | 75 +++++---
pkg/cmd/util_sources.go | 201 +++++++++++++++++++++
pkg/util/util.go | 12 ++
11 files changed, 448 insertions(+), 190 deletions(-)
diff --git a/docs/modules/ROOT/pages/running/run-from-github.adoc b/docs/modules/ROOT/pages/running/run-from-github.adoc
index 5f071b6..1d606b8 100644
--- a/docs/modules/ROOT/pages/running/run-from-github.adoc
+++ b/docs/modules/ROOT/pages/running/run-from-github.adoc
@@ -1,25 +1,64 @@
[[run-from-github]]
= Run from GitHub
-It is possible to run integrations from GitHub with a dedicated URL
-syntax:
+It is possible to run integrations from a GitHub repository or Gist with dedicated URL syntax:
-```
+== Repository
+
+.Syntax
+[source]
+----
kamel run github:$user/$repo/$path?branch=$branch
-```
+----
As example, running the following command
-```
+
+[source]
+----
kamel run github:apache/camel-k/examples/Sample.java
-```
+----
is equivalent to:
-```
+[source]
+----
kamel run https://raw.githubusercontent.com/apache/camel-k/master/examples/Sample.java
-```
+----
but does not require to type the full GitHub RAW URL.
-Declaring the branch query param is not required and defaults to `master` if not explicit set.
\ No newline at end of file
+Declaring the branch query param is not required and defaults to `master` if not explicit set.
+
+== Gist
+
+.Syntax
+[source]
+----
+kamel run https://gist.github.com/${user-id}/${gist-id}
+kamel run gist:${gist-id}
+----
+
+camel-k will add any file that is part of the Gist as a source.
+
+As example, assuming there are two files listed as part of a Gist, beans.yaml and routes.yaml, then the following command
+
+
+[source]
+----
+kamel run gist:${gist-id}
+----
+
+is equivalent to:
+
+[source]
+----
+kamel run \
+ https://gist.githubusercontent.com/${user-id}/${gist-id}/raw/${...}/beans.yaml \
+ https://gist.githubusercontent.com/${user-id}/${gist-id}/raw/${...}/routes.yaml
+----
+
+[NOTE]
+====
+GitHub applies rate limiting to its APIs and as Authenticated requests get a higher rate limit, the camel-k cli honour the env var GITHUB_TOKEN and if it is found, then it is used for GitHub authentication.
+====
\ No newline at end of file
diff --git a/e2e/common/run_test.go b/e2e/common/run_test.go
index 1309bf8..3cb833d 100644
--- a/e2e/common/run_test.go
+++ b/e2e/common/run_test.go
@@ -44,24 +44,6 @@ func TestRunSimpleExamples(t *testing.T) {
Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
})
- t.Run("run java from GitHub", func(t *testing.T) {
- RegisterTestingT(t)
- Expect(Kamel("run", "-n", ns, "github:apache/camel-k/e2e/common/files/Java.java").Execute()).Should(BeNil())
- Eventually(IntegrationPodPhase(ns, "java"), TestTimeoutMedium).Should(Equal(v1.PodRunning))
- Eventually(IntegrationCondition(ns, "java", camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
- Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
- Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
- })
-
- t.Run("run java from GitHub (RAW)", func(t *testing.T) {
- RegisterTestingT(t)
- Expect(Kamel("run", "-n", ns, "https://raw.githubusercontent.com/apache/camel-k/master/e2e/common/files/Java.java").Execute()).Should(BeNil())
- Eventually(IntegrationPodPhase(ns, "java"), TestTimeoutMedium).Should(Equal(v1.PodRunning))
- Eventually(IntegrationCondition(ns, "java", camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
- Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
- Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
- })
-
t.Run("run java with properties", func(t *testing.T) {
RegisterTestingT(t)
Expect(Kamel("run", "-n", ns, "files/Prop.java", "--property-file", "files/prop.properties").Execute()).Should(BeNil())
@@ -138,3 +120,49 @@ func TestRunSimpleExamples(t *testing.T) {
})
}
+
+func TestRunExamplesFromGitHUB(t *testing.T) {
+ WithNewTestNamespace(t, func(ns string) {
+ Expect(Kamel("install", "-n", ns).Execute()).Should(BeNil())
+
+ t.Run("run java from GitHub", func(t *testing.T) {
+ RegisterTestingT(t)
+ Expect(Kamel("run", "-n", ns, "github:apache/camel-k/e2e/common/files/Java.java").Execute()).Should(BeNil())
+ Eventually(IntegrationPodPhase(ns, "java"), TestTimeoutMedium).Should(Equal(v1.PodRunning))
+ Eventually(IntegrationCondition(ns, "java", camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
+ Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+ Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
+ })
+
+ t.Run("run java from GitHub (RAW)", func(t *testing.T) {
+ RegisterTestingT(t)
+ Expect(Kamel("run", "-n", ns, "https://raw.githubusercontent.com/apache/camel-k/master/e2e/common/files/Java.java").Execute()).Should(BeNil())
+ Eventually(IntegrationPodPhase(ns, "java"), TestTimeoutMedium).Should(Equal(v1.PodRunning))
+ Eventually(IntegrationCondition(ns, "java", camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
+ Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+ Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
+ })
+
+ t.Run("run from GitHub Gist (ID)", func(t *testing.T) {
+ name := "github-gist-id"
+ RegisterTestingT(t)
+ Expect(Kamel("run", "-n", ns, "--name", name, "gist:e2c3f9a5fd0d9e79b21b04809786f17a").Execute()).Should(BeNil())
+ Eventually(IntegrationPodPhase(ns, name), TestTimeoutMedium).Should(Equal(v1.PodRunning))
+ Eventually(IntegrationCondition(ns, name, camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
+ Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+ Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Tick!"))
+ Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
+ })
+
+ t.Run("run from GitHub Gist (URL)", func(t *testing.T) {
+ name := "github-gist-url"
+ RegisterTestingT(t)
+ Expect(Kamel("run", "-n", ns, "--name", name, "https://gist.github.com/lburgazzoli/e2c3f9a5fd0d9e79b21b04809786f17a").Execute()).Should(BeNil())
+ Eventually(IntegrationPodPhase(ns, name), TestTimeoutMedium).Should(Equal(v1.PodRunning))
+ Eventually(IntegrationCondition(ns, name, camelv1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(v1.ConditionTrue))
+ Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))
+ Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Tick!"))
+ Expect(Kamel("delete", "--all", "-n", ns).Execute()).Should(BeNil())
+ })
+ })
+}
diff --git a/go.mod b/go.mod
index b0fe8b6..2a218d4 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,7 @@ require (
github.com/fatih/structs v1.1.0
github.com/gertd/go-pluralize v0.1.1
github.com/go-logr/logr v0.1.0
+ github.com/google/go-github/v32 v32.1.0
github.com/google/uuid v1.1.1
github.com/jpillora/backoff v1.0.0
github.com/magiconair/properties v1.8.1
@@ -35,6 +36,7 @@ require (
github.com/stoewer/go-strcase v1.0.2
github.com/stretchr/testify v1.5.1
go.uber.org/multierr v1.5.0
+ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
gopkg.in/inf.v0 v0.9.1
gopkg.in/yaml.v2 v2.3.0
k8s.io/api v0.18.9
diff --git a/go.sum b/go.sum
index 7d03bdb..20d9cc2 100644
--- a/go.sum
+++ b/go.sum
@@ -585,11 +585,15 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-github/v27 v27.0.6/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
github.com/google/go-github/v29 v29.0.3/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
+github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
+github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk=
github.com/google/go-licenses v0.0.0-20200227160636-0fa8c766a591/go.mod h1:JWeTIGPLQ9gF618ZOdlUitd1gRR/l99WOkHOlmR/UVA=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE=
github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no=
diff --git a/pkg/cmd/modeline.go b/pkg/cmd/modeline.go
index 1e1ca37..4eab57a 100644
--- a/pkg/cmd/modeline.go
+++ b/pkg/cmd/modeline.go
@@ -47,7 +47,6 @@ var (
// file options must be considered relative to the source files they belong to
fileOptions = map[string]bool{
- "source": true,
"resource": true,
"config": true,
"open-api": true,
@@ -55,10 +54,10 @@ var (
}
)
+// NewKamelWithModelineCommand ---
func NewKamelWithModelineCommand(ctx context.Context, osArgs []string) (*cobra.Command, []string, error) {
- processed := make(map[string]bool)
originalFlags := osArgs[1:]
- rootCmd, flags, err := createKamelWithModelineCommand(ctx, append([]string(nil), originalFlags...), processed)
+ rootCmd, flags, err := createKamelWithModelineCommand(ctx, originalFlags)
if err != nil {
fmt.Printf("Error: %s\n", err.Error())
return rootCmd, flags, err
@@ -75,7 +74,7 @@ func NewKamelWithModelineCommand(ctx context.Context, osArgs []string) (*cobra.C
return rootCmd, flags, nil
}
-func createKamelWithModelineCommand(ctx context.Context, args []string, processedFiles map[string]bool) (*cobra.Command, []string, error) {
+func createKamelWithModelineCommand(ctx context.Context, args []string) (*cobra.Command, []string, error) {
rootCmd, err := NewKamelCommand(ctx)
if err != nil {
return nil, nil, err
@@ -99,44 +98,20 @@ func createKamelWithModelineCommand(ctx context.Context, args []string, processe
fg := target.Flags()
- sources, err := fg.GetStringArray(runCmdSourcesArgs)
+ additionalSources, err := fg.GetStringArray(runCmdSourcesArgs)
if err != nil {
return nil, nil, err
}
- var files = append([]string(nil), fg.Args()...)
- files = append(files, sources...)
+ files := make([]string, 0, len(fg.Args())+len(additionalSources))
+ files = append(files, fg.Args()...)
+ files = append(files, additionalSources...)
- opts := make([]modeline.Option, 0)
- for _, f := range files {
- if processedFiles[f] {
- continue
- }
- baseDir := filepath.Dir(f)
- content, _, err := loadData(f, false, false)
- if err != nil {
- return nil, nil, errors.Wrapf(err, "cannot read file %s", f)
- }
- ops, err := modeline.Parse(f, content)
- if err != nil {
- return nil, nil, errors.Wrapf(err, "cannot process file %s", f)
- }
- for i, o := range ops {
- if disallowedOptions[o.Name] {
- return nil, nil, fmt.Errorf("option %q is disallowed in modeline", o.Name)
- }
-
- if fileOptions[o.Name] && isLocal(f) {
- refPath := o.Value
- if !filepath.IsAbs(refPath) {
- full := path.Join(baseDir, refPath)
- o.Value = full
- ops[i] = o
- }
- }
- }
- opts = append(opts, ops...)
+ opts, err := extractModelineOptions(ctx, files)
+ if err != nil {
+ return nil, nil, errors.Wrap(err, "cannot read sources")
}
+
// filter out in place non-run options
nOpts := 0
for _, o := range opts {
@@ -145,23 +120,9 @@ func createKamelWithModelineCommand(ctx context.Context, args []string, processe
nOpts++
}
}
- opts = opts[:nOpts]
- // No new options, returning a new command with computed args
- if len(opts) == 0 {
- // Recreating the command as it's dirty
- rootCmd, err = NewKamelCommand(ctx)
- if err != nil {
- return nil, nil, err
- }
- rootCmd.SetArgs(args)
- return rootCmd, args, nil
- }
+ opts = opts[:nOpts]
- // New options added, recomputing
- for _, f := range files {
- processedFiles[f] = true
- }
for _, o := range opts {
prefix := "-"
if len(o.Name) > 1 {
@@ -175,5 +136,46 @@ func createKamelWithModelineCommand(ctx context.Context, args []string, processe
}
}
- return createKamelWithModelineCommand(ctx, args, processedFiles)
+ // Recreating the command as it's dirty
+ rootCmd, err = NewKamelCommand(ctx)
+ if err != nil {
+ return nil, nil, err
+ }
+ rootCmd.SetArgs(args)
+
+ return rootCmd, args, nil
+}
+
+func extractModelineOptions(ctx context.Context, sources []string) ([]modeline.Option, error) {
+ opts := make([]modeline.Option, 0)
+
+ resolvedSources, err := ResolveSources(ctx, sources, false)
+ if err != nil {
+ return opts, errors.Wrap(err, "cannot read sources")
+ }
+
+ for _, resolvedSource := range resolvedSources {
+ ops, err := modeline.Parse(resolvedSource.Location, resolvedSource.Content)
+ if err != nil {
+ return opts, errors.Wrapf(err, "cannot process file %s", resolvedSource.Location)
+ }
+ for i, o := range ops {
+ if disallowedOptions[o.Name] {
+ return opts, fmt.Errorf("option %q is disallowed in modeline", o.Name)
+ }
+
+ if fileOptions[o.Name] && resolvedSource.Local {
+ baseDir := filepath.Dir(resolvedSource.Origin)
+ refPath := o.Value
+ if !filepath.IsAbs(refPath) {
+ full := path.Join(baseDir, refPath)
+ o.Value = full
+ ops[i] = o
+ }
+ }
+ }
+ opts = append(opts, ops...)
+ }
+
+ return opts, nil
}
diff --git a/pkg/cmd/modeline_test.go b/pkg/cmd/modeline_test.go
index 20e91bd..ca6037a 100644
--- a/pkg/cmd/modeline_test.go
+++ b/pkg/cmd/modeline_test.go
@@ -82,23 +82,23 @@ func TestModelineRunMultipleFiles(t *testing.T) {
defer os.RemoveAll(dir)
file := `
- // camel-k: source=ext.groovy
+ // camel-k: dependency=mvn:org.my/lib1:3.0
`
fileName := path.Join(dir, "simple.groovy")
err = ioutil.WriteFile(fileName, []byte(file), 0777)
assert.NoError(t, err)
file2 := `
- // camel-k: dependency=mvn:org.my/lib:3.0
+ // camel-k: dependency=mvn:org.my/lib2:3.0
`
fileName2 := path.Join(dir, "ext.groovy")
err = ioutil.WriteFile(fileName2, []byte(file2), 0777)
assert.NoError(t, err)
- cmd, flags, err := NewKamelWithModelineCommand(context.TODO(), []string{"kamel", "run", fileName})
+ cmd, flags, err := NewKamelWithModelineCommand(context.TODO(), []string{"kamel", "run", fileName, fileName2})
assert.NoError(t, err)
assert.NotNil(t, cmd)
- assert.Equal(t, []string{"run", fileName, "--source=" + fileName2, "--dependency=mvn:org.my/lib:3.0"}, flags)
+ assert.Equal(t, []string{"run", fileName, fileName2, "--dependency=mvn:org.my/lib1:3.0", "--dependency=mvn:org.my/lib2:3.0"}, flags)
}
func TestModelineRunPropertyFiles(t *testing.T) {
diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go
index db2c7c3..c515e77 100644
--- a/pkg/cmd/run.go
+++ b/pkg/cmd/run.go
@@ -18,13 +18,10 @@ limitations under the License.
package cmd
import (
- "bytes"
- "encoding/base64"
+ "context"
"encoding/json"
"fmt"
"io/ioutil"
- "net/http"
- "net/url"
"os"
"os/signal"
"path"
@@ -49,7 +46,6 @@ import (
"github.com/apache/camel-k/pkg/trait"
"github.com/apache/camel-k/pkg/util"
"github.com/apache/camel-k/pkg/util/flow"
- "github.com/apache/camel-k/pkg/util/gzip"
"github.com/apache/camel-k/pkg/util/kubernetes"
k8slog "github.com/apache/camel-k/pkg/util/kubernetes/log"
"github.com/apache/camel-k/pkg/util/sync"
@@ -207,19 +203,8 @@ func (o *runCmdOptions) validateArgs(_ *cobra.Command, args []string) error {
return errors.New("run expects at least 1 argument, received 0")
}
- for _, source := range args {
- if isLocal(source) {
- if _, err := os.Stat(source); err != nil && os.IsNotExist(err) {
- return errors.Wrapf(err, "file %s does not exist", source)
- } else if err != nil {
- return errors.Wrapf(err, "error while accessing file %s", source)
- }
- } else {
- _, _, err := loadData(source, false, false)
- if err != nil {
- return errors.Wrap(err, "The provided source is not reachable")
- }
- }
+ if _, err := ResolveSources(context.Background(), args, false); err != nil {
+ return errors.Wrap(err, "One of the provided sources is not reachable")
}
return nil
@@ -414,7 +399,7 @@ func (o *runCmdOptions) syncIntegration(cmd *cobra.Command, c client.Client, sou
return
case <-changes:
// let's create a new command to parse modeline changes and update our integration
- newCmd, _, err := createKamelWithModelineCommand(o.RootContext, os.Args[1:], make(map[string]bool))
+ newCmd, _, err := createKamelWithModelineCommand(o.RootContext, os.Args[1:])
newCmd.SetOut(cmd.OutOrStdout())
newCmd.SetErr(cmd.ErrOrStderr())
if err != nil {
@@ -493,14 +478,14 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string,
srcs = append(srcs, sources...)
srcs = append(srcs, o.Sources...)
- for _, source := range srcs {
- data, compressed, err := loadData(source, o.Compression, o.CompressBinary)
- if err != nil {
- return nil, err
- }
+ resolvedSources, err := ResolveSources(context.Background(), srcs, o.Compression)
+ if err != nil {
+ return nil, err
+ }
- if !compressed && o.UseFlows && (strings.HasSuffix(source, ".yaml") || strings.HasSuffix(source, ".yml")) {
- flows, err := flow.FromYamlDSLString(data)
+ for _, source := range resolvedSources {
+ if o.UseFlows && !o.Compression && (strings.HasSuffix(source.Name, ".yaml") || strings.HasSuffix(source.Name, ".yml")) {
+ flows, err := flow.FromYamlDSLString(source.Content)
if err != nil {
return nil, err
}
@@ -508,16 +493,16 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string,
} else {
integration.Spec.AddSources(v1.SourceSpec{
DataSpec: v1.DataSpec{
- Name: path.Base(source),
- Content: data,
- Compression: compressed,
+ Name: source.Name,
+ Content: source.Content,
+ Compression: source.Compress,
},
})
}
}
for _, resource := range o.Resources {
- data, compressed, err := loadData(resource, o.Compression, o.CompressBinary)
+ data, compressed, err := loadContent(resource, o.Compression, o.CompressBinary)
if err != nil {
return nil, err
}
@@ -533,7 +518,7 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string,
}
for _, resource := range o.OpenAPIs {
- data, compressed, err := loadData(resource, o.Compression, o.CompressBinary)
+ data, compressed, err := loadContent(resource, o.Compression, o.CompressBinary)
if err != nil {
return nil, err
}
@@ -603,7 +588,7 @@ func (o *runCmdOptions) updateIntegrationCode(c client.Client, sources []string,
}
existed := false
- err := c.Create(o.Context, &integration)
+ err = c.Create(o.Context, &integration)
if err != nil && k8serrors.IsAlreadyExists(err) {
existed = true
clone := integration.DeepCopy()
@@ -656,53 +641,6 @@ func (o *runCmdOptions) GetIntegrationName(sources []string) string {
return name
}
-func loadData(source string, compress bool, compressBinary bool) (string, bool, error) {
- var content []byte
- var err error
-
- if isLocal(source) {
- content, err = ioutil.ReadFile(source)
- if err != nil {
- return "", false, err
- }
- } else {
- u, err := url.Parse(source)
- if err != nil {
- return "", false, err
- }
-
- g, ok := Getters[u.Scheme]
- if !ok {
- return "", false, fmt.Errorf("unable to find a getter for URL: %s", source)
- }
-
- content, err = g.Get(u)
- if err != nil {
- return "", false, err
- }
- }
-
- doCompress := compress
- if !doCompress && compressBinary {
- contentType := http.DetectContentType(content)
- if strings.HasPrefix(contentType, "application/octet-stream") {
- doCompress = true
- }
- }
-
- if doCompress {
- var b bytes.Buffer
-
- if err := gzip.Compress(&b, content); err != nil {
- return "", false, err
- }
-
- return base64.StdEncoding.EncodeToString(b.Bytes()), true, nil
- }
-
- return string(content), false, nil
-}
-
func (*runCmdOptions) configureTraits(integration *v1.Integration, options []string, catalog *trait.Catalog) error {
traits, err := configureTraits(options, catalog)
if err != nil {
diff --git a/pkg/cmd/util.go b/pkg/cmd/util.go
index 6cfff5f..8d5ac14 100644
--- a/pkg/cmd/util.go
+++ b/pkg/cmd/util.go
@@ -18,10 +18,13 @@ limitations under the License.
package cmd
import (
+ "bytes"
"context"
+ "encoding/base64"
"encoding/csv"
"encoding/json"
"fmt"
+ "github.com/apache/camel-k/pkg/util/gzip"
"log"
"reflect"
"strings"
@@ -237,3 +240,13 @@ func fieldByMapstructureTagName(target reflect.Value, tagName string) (reflect.S
return reflect.StructField{}, false
}
+
+func compressToString(content []byte) (string, error) {
+ var b bytes.Buffer
+
+ if err := gzip.Compress(&b, content); err != nil {
+ return "", err
+ }
+
+ return base64.StdEncoding.EncodeToString(b.Bytes()), nil
+}
diff --git a/pkg/cmd/util_getter.go b/pkg/cmd/util_content.go
similarity index 56%
rename from pkg/cmd/util_getter.go
rename to pkg/cmd/util_content.go
index 4be7526..75d9c08 100644
--- a/pkg/cmd/util_getter.go
+++ b/pkg/cmd/util_content.go
@@ -23,34 +23,55 @@ import (
"net/http"
"net/url"
"regexp"
+ "strings"
)
-var Getters map[string]Getter
-
-func init() {
- Getters = map[string]Getter{
- "http": HTTPGetter{},
- "https": HTTPGetter{},
- "github": GitHubGetter{},
+func loadContent(source string, compress bool, compressBinary bool) (string, bool, error) {
+ var content []byte
+ var err error
+
+ if isLocal(source) {
+ content, err = ioutil.ReadFile(source)
+ } else {
+ u, err := url.Parse(source)
+ if err != nil {
+ return "", false, err
+ }
+
+ switch u.Scheme {
+ case "github":
+ content, err = loadContentGitHub(u)
+ case "http":
+ content, err = loadContentHTTP(u)
+ case "https":
+ content, err = loadContentHTTP(u)
+ default:
+ return "", false, fmt.Errorf("unsupported scheme %s", u.Scheme)
+ }
}
-}
-type Getter interface {
- Get(u *url.URL) ([]byte, error)
-}
+ if err != nil {
+ return "", false, err
+ }
+ doCompress := compress
+ if !doCompress && compressBinary {
+ contentType := http.DetectContentType(content)
+ if strings.HasPrefix(contentType, "application/octet-stream") {
+ doCompress = true
+ }
+ }
-// A simple getter that retrieves the content of an integration from an
-// http(s) endpoint.
-type HTTPGetter struct {
-}
+ if doCompress {
+ answer, err := compressToString(content)
+ return answer, true, err
+ }
-func (g HTTPGetter) Get(u *url.URL) ([]byte, error) {
- return g.doGet(u.String())
+ return string(content), false, nil
}
-func (g HTTPGetter) doGet(source string) ([]byte, error) {
+func loadContentHTTP(u *url.URL) ([]byte, error) {
// nolint: gosec
- resp, err := http.Get(source)
+ resp, err := http.Get(u.String())
if err != nil {
return []byte{}, err
}
@@ -59,7 +80,7 @@ func (g HTTPGetter) doGet(source string) ([]byte, error) {
}()
if resp.StatusCode != 200 {
- return []byte{}, fmt.Errorf("the provided URL %s is not reachable, error code is %d", source, resp.StatusCode)
+ return []byte{}, fmt.Errorf("the provided URL %s is not reachable, error code is %d", u.String(), resp.StatusCode)
}
content, err := ioutil.ReadAll(resp.Body)
@@ -70,13 +91,7 @@ func (g HTTPGetter) doGet(source string) ([]byte, error) {
return content, nil
}
-// A simple getter that retrieves the content of an integration from
-// a GitHub endpoint using a RAW endpoint.
-type GitHubGetter struct {
- HTTPGetter
-}
-
-func (g GitHubGetter) Get(u *url.URL) ([]byte, error) {
+func loadContentGitHub(u *url.URL) ([]byte, error) {
src := u.Scheme + ":" + u.Opaque
re := regexp.MustCompile(`^github:([^/]+)/([^/]+)/(.+)$`)
@@ -91,6 +106,10 @@ func (g GitHubGetter) Get(u *url.URL) ([]byte, error) {
}
srcURL := fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", items[1], items[2], branch, items[3])
+ rawURL, err := url.Parse(srcURL)
+ if err != nil {
+ return []byte{}, err
+ }
- return g.HTTPGetter.doGet(srcURL)
+ return loadContentHTTP(rawURL)
}
diff --git a/pkg/cmd/util_sources.go b/pkg/cmd/util_sources.go
new file mode 100644
index 0000000..af218f0
--- /dev/null
+++ b/pkg/cmd/util_sources.go
@@ -0,0 +1,201 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "os"
+ "path"
+ "strings"
+
+ "github.com/apache/camel-k/pkg/util"
+
+ "golang.org/x/oauth2"
+
+ "github.com/google/go-github/v32/github"
+ "github.com/pkg/errors"
+)
+
+// Source ---
+type Source struct {
+ Origin string
+ Location string
+ Name string
+ Content string
+ Compress bool
+ Local bool
+}
+
+func (s *Source) setContent(content []byte) error {
+ if s.Compress {
+ result, err := compressToString(content)
+ if err != nil {
+ return err
+ }
+
+ s.Content = result
+ } else {
+ s.Content = string(content)
+ }
+
+ return nil
+}
+
+// ResolveSources ---
+func ResolveSources(ctx context.Context, locations []string, compress bool) ([]Source, error) {
+ sources := make([]Source, 0, len(locations))
+
+ for _, location := range locations {
+ if isLocal(location) {
+ if _, err := os.Stat(location); err != nil && os.IsNotExist(err) {
+ return sources, errors.Wrapf(err, "file %s does not exist", location)
+ } else if err != nil {
+ return sources, errors.Wrapf(err, "error while accessing file %s", location)
+ }
+
+ answer := Source{
+ Name: path.Base(location),
+ Origin: location,
+ Location: location,
+ Compress: compress,
+ Local: true,
+ }
+
+ content, err := ioutil.ReadFile(location)
+ if err != nil {
+ return sources, err
+ }
+ if err := answer.setContent(content); err != nil {
+ return sources, err
+ }
+
+ sources = append(sources, answer)
+ } else {
+ u, err := url.Parse(location)
+ if err != nil {
+ return sources, err
+ }
+
+ switch {
+ case u.Scheme == "gist" || strings.HasPrefix(location, "https://gist.github.com/"):
+ var tc *http.Client
+
+ if token, ok := os.LookupEnv("GITHUB_TOKEN"); ok {
+ ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
+ tc = oauth2.NewClient(ctx, ts)
+
+ fmt.Println("GITHUB_TOKEN env var detected, using it for GitHub APIs authentication")
+ }
+
+ gc := github.NewClient(tc)
+ gistID := ""
+
+ if strings.HasPrefix(location, "https://gist.github.com/") {
+ names := util.FindNamedMatches(`^https://gist.github.com/(([a-zA-Z0-9]*)/)?(?P<gistid>[a-zA-Z0-9]*)$`, location)
+ if value, ok := names["gistid"]; ok {
+ gistID = value
+ }
+ } else {
+ gistID = u.Opaque
+ }
+
+ if gistID == "" {
+ return sources, fmt.Errorf("unable to determing gist id from %s", location)
+ }
+
+ gists, _, err := gc.Gists.Get(ctx, gistID)
+ if err != nil {
+ return sources, err
+ }
+
+ for _, v := range gists.Files {
+ if v.Filename == nil || v.Content == nil {
+ continue
+ }
+
+ answer := Source{
+ Name: *v.Filename,
+ Compress: compress,
+ Origin: location,
+ }
+ if v.RawURL != nil {
+ answer.Location = *v.RawURL
+ }
+ if err := answer.setContent([]byte(*v.Content)); err != nil {
+ return sources, err
+ }
+ sources = append(sources, answer)
+ }
+ case u.Scheme == "github":
+ answer := Source{
+ Name: path.Base(location),
+ Origin: location,
+ Location: location,
+ Compress: compress,
+ }
+
+ content, err := loadContentGitHub(u)
+ if err != nil {
+ return sources, err
+ }
+ if err := answer.setContent(content); err != nil {
+ return sources, err
+ }
+ sources = append(sources, answer)
+ case u.Scheme == "http":
+ answer := Source{
+ Name: path.Base(location),
+ Origin: location,
+ Location: location,
+ Compress: compress,
+ }
+
+ content, err := loadContentHTTP(u)
+ if err != nil {
+ return sources, err
+ }
+ if err := answer.setContent(content); err != nil {
+ return sources, err
+ }
+ sources = append(sources, answer)
+ case u.Scheme == "https":
+ answer := Source{
+ Name: path.Base(location),
+ Origin: location,
+ Location: location,
+ Compress: compress,
+ }
+
+ content, err := loadContentHTTP(u)
+ if err != nil {
+ return sources, err
+ }
+ if err := answer.setContent(content); err != nil {
+ return sources, err
+ }
+ sources = append(sources, answer)
+ }
+ }
+ }
+
+ return sources, nil
+}
diff --git a/pkg/util/util.go b/pkg/util/util.go
index cfd7e18..1867126 100644
--- a/pkg/util/util.go
+++ b/pkg/util/util.go
@@ -209,6 +209,18 @@ func FindAllDistinctStringSubmatch(data string, regexps ...*regexp.Regexp) []str
return submatchs.List()
}
+// FindNamedMatches ---
+func FindNamedMatches(expr string, str string) map[string]string {
+ regex := regexp.MustCompile(expr)
+ match := regex.FindStringSubmatch(str)
+
+ results := map[string]string{}
+ for i, name := range match {
+ results[regex.SubexpNames()[i]] = name
+ }
+ return results
+}
+
// FileExists --
func FileExists(name string) (bool, error) {
info, err := os.Stat(name)