You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/03/27 21:26:17 UTC
[GitHub] mrutkows closed pull request #813: Provides support to export projects with dependencies
mrutkows closed pull request #813: Provides support to export projects with dependencies
URL: https://github.com/apache/incubator-openwhisk-wskdeploy/pull/813
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/cmd/export.go b/cmd/export.go
index 437fb4f2..b0518d8f 100644
--- a/cmd/export.go
+++ b/cmd/export.go
@@ -44,18 +44,7 @@ var exportCmd = &cobra.Command{
var config *whisk.Config
-func ExportRule(wskRule whisk.Rule, pkgName string, maniyaml *parsers.YAML) {
- if maniyaml.Packages[pkgName].Rules == nil {
- pkg := maniyaml.Packages[pkgName]
- pkg.Rules = make(map[string]parsers.Rule)
- maniyaml.Packages[pkgName] = pkg
- }
-
- // export rule to manifest
- maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(wskRule)
-}
-
-func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML) error {
+func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML, targetManifest string) error {
pkg := maniyaml.Packages[packageName]
if pkg.Actions == nil {
@@ -71,7 +60,7 @@ func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML)
seq := new(parsers.Sequence)
for _, component := range wskAction.Exec.Components {
// must ommit namespace from seq component name
- ExportAction(strings.SplitN(component, "/", 3)[2], packageName, maniyaml)
+ ExportAction(strings.SplitN(component, "/", 3)[2], packageName, maniyaml, targetManifest)
slices := strings.Split(component, "/")
// save in the seq list only action names
@@ -90,7 +79,7 @@ func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML)
pkg.Sequences[wskAction.Name] = *seq
} else {
parsedAction := *maniyaml.ComposeParsersAction(*wskAction)
- manifestDir := filepath.Dir(utils.Flags.ManifestPath)
+ manifestDir := filepath.Dir(targetManifest)
// store function file under action package name subdirectory in the specified manifest folder
functionDir := filepath.Join(manifestDir, packageName)
@@ -111,24 +100,18 @@ func ExportAction(actionName string, packageName string, maniyaml *parsers.YAML)
return nil
}
-func ExportCmdImp(cmd *cobra.Command, args []string) error {
-
- projectName := utils.Flags.ProjectPath
+func exportProject(projectName string, targetManifest string) error {
maniyaml := &parsers.YAML{}
maniyaml.Project.Name = projectName
- config, _ = deployers.NewWhiskConfig(wskpropsPath, utils.Flags.DeploymentPath, utils.Flags.ManifestPath)
- client, _ = deployers.CreateNewClient(config)
-
- // Init supported runtimes and action files extensions maps
- setSupportedRuntimes(config.Host)
-
// Get the list of packages in your namespace
packages, _, err := client.Packages.List(&whisk.PackageListOptions{})
if err != nil {
return err
}
+ var bindings = make(map[string]whisk.Binding)
+
// iterate over each package to find managed annotations
// check if "managed" annotation is attached to a package
// add to export when managed project name matches with the
@@ -141,6 +124,12 @@ func ExportCmdImp(cmd *cobra.Command, args []string) error {
// we have found a package which is part of the current project
if pa[utils.OW_PROJECT_NAME] == projectName {
+ // check if the package is dependency
+ if pkg.Annotations.GetValue(utils.BINDING) != nil {
+ bindings[pkg.Name] = *pkg.Binding
+ continue
+ }
+
if maniyaml.Packages == nil {
maniyaml.Packages = make(map[string]parsers.Package)
}
@@ -172,7 +161,7 @@ func ExportCmdImp(cmd *cobra.Command, args []string) error {
if aa[utils.OW_PROJECT_NAME] == projectName {
actionName := strings.Join([]string{pkg.Name, action.Name}, "/")
// export action to file system
- err = ExportAction(actionName, pkg.Name, maniyaml)
+ err = ExportAction(actionName, pkg.Name, maniyaml, targetManifest)
if err != nil {
return err
}
@@ -220,49 +209,100 @@ func ExportCmdImp(cmd *cobra.Command, args []string) error {
return err
}
- // TODO: can be simplifyed once OW permits to add annotation to rules
- // iterate over the list of rules to determine whether any of them is part of
- // managed trigger -> action set for specified managed project. if yes, add to manifest
+ // iterate over the list of rules to determine whether any of them is part of the manage dproject
for _, rule := range rules {
// get rule from OW
wskRule, _, _ := client.Rules.Get(rule.Name)
- ruleAction := wskRule.Action.(map[string]interface{})["name"].(string)
- ruleTrigger := wskRule.Trigger.(map[string]interface{})["name"].(string)
-
- // can be simplified once rules moved to top level next to triggers
- for pkgName := range maniyaml.Packages {
- if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace {
- // iterate over all managed triggers in manifest
- for _, trigger := range maniyaml.Packages[pkgName].Triggers {
- // check that managed trigger equals to rule trigger
- if ruleTrigger == trigger.Name && trigger.Namespace == wskRule.Namespace {
- // check that there a managed action (or sequence action) equals to rule action
- for _, action := range maniyaml.Packages[pkgName].Actions {
- if action.Name == ruleAction {
- // export rule to manifest
- ExportRule(*wskRule, pkgName, maniyaml)
- }
- }
+ // rule has attached managed annotation
+ if a := wskRule.Annotations.GetValue(utils.MANAGED); a != nil {
+ // decode the JSON blob and retrieve __OW_PROJECT_NAME
+ ta := a.(map[string]interface{})
+ if ta[utils.OW_PROJECT_NAME] == projectName {
- // check that there a managed sequence action with name matching the rule action
- for name := range maniyaml.Packages[pkgName].Sequences {
- if name == ruleAction {
- // export rule to manifest
- ExportRule(*wskRule, pkgName, maniyaml)
- }
+ for pkgName := range maniyaml.Packages {
+ if maniyaml.Packages[pkgName].Namespace == wskRule.Namespace {
+ if maniyaml.Packages[pkgName].Rules == nil {
+ pkg := maniyaml.Packages[pkgName]
+ pkg.Rules = make(map[string]parsers.Rule)
+ maniyaml.Packages[pkgName] = pkg
}
+
+ // export rule to manifest
+ maniyaml.Packages[pkgName].Rules[wskRule.Name] = *maniyaml.ComposeParsersRule(*wskRule)
}
}
}
}
+
+ }
+
+ // adding dependencies to the first package
+ for pkgName := range maniyaml.Packages {
+ for bPkg, binding := range bindings {
+ if maniyaml.Packages[pkgName].Dependencies == nil {
+ pkg := maniyaml.Packages[pkgName]
+ pkg.Dependencies = make(map[string]parsers.Dependency)
+ maniyaml.Packages[pkgName] = pkg
+ }
+ maniyaml.Packages[pkgName].Dependencies[bPkg] = *maniyaml.ComposeParsersDependency(binding)
+ }
+
+ break
+ }
+
+ // export manifest to file
+ parsers.Write(maniyaml, targetManifest)
+ fmt.Println("Manifest exported to: " + targetManifest)
+
+ // find exported manifest parent directory
+ manifestDir := filepath.Dir(utils.Flags.ManifestPath)
+
+ // create dependencies directory if not exists
+ depDir := filepath.Join(manifestDir, "dependencies")
+
+ if len(bindings) > 0 {
+ fmt.Println("Exporting project dependencies to " + depDir)
}
- parsers.Write(maniyaml, utils.Flags.ManifestPath)
- fmt.Println("Manifest exported to: " + utils.Flags.ManifestPath)
+ // now export dependencies to their own manifests
+ for _, binding := range bindings {
+ pkg, _, err := client.Packages.Get(binding.Name)
+ if err != nil {
+ return err
+ }
+
+ if a := pkg.Annotations.GetValue(utils.MANAGED); a != nil {
+ // decode the JSON blob and retrieve __OW_PROJECT_NAME
+ pa := a.(map[string]interface{})
+
+ os.MkdirAll(depDir, os.ModePerm)
+ depManifestPath := filepath.Join(depDir, pa[utils.OW_PROJECT_NAME].(string)+".yaml")
+
+ // export the whole project as dependency
+ err := exportProject(pa[utils.OW_PROJECT_NAME].(string), depManifestPath)
+ if err != nil {
+ return err
+ }
+ } else {
+ // showing warning to notify user that exported manifest dependent on unmanaged library which can't be exported
+ fmt.Println("Warning! Dependency package " + binding.Name + " currently unmanaged by any project. Unable to export this package")
+ }
+ }
return nil
}
+func ExportCmdImp(cmd *cobra.Command, args []string) error {
+
+ config, _ = deployers.NewWhiskConfig(wskpropsPath, utils.Flags.DeploymentPath, utils.Flags.ManifestPath)
+ client, _ = deployers.CreateNewClient(config)
+
+ // Init supported runtimes and action files extensions maps
+ setSupportedRuntimes(config.Host)
+
+ return exportProject(utils.Flags.ProjectPath, utils.Flags.ManifestPath)
+}
+
const (
JAVA_EXT = ".jar"
ZIP_EXT = ".zip"
diff --git a/deployers/manifestreader.go b/deployers/manifestreader.go
index fa541169..692b4036 100644
--- a/deployers/manifestreader.go
+++ b/deployers/manifestreader.go
@@ -68,7 +68,7 @@ func (deployer *ManifestReader) HandleYaml(sdeployer *ServiceDeployer, manifestP
var err error
var manifestName = manifest.Filepath
- deps, err := manifestParser.ComposeDependenciesFromAllPackages(manifest, deployer.serviceDeployer.ProjectPath, deployer.serviceDeployer.ManifestPath)
+ deps, err := manifestParser.ComposeDependenciesFromAllPackages(manifest, deployer.serviceDeployer.ProjectPath, deployer.serviceDeployer.ManifestPath, ma)
if err != nil {
return wskderrors.NewYAMLFileFormatError(manifestName, err)
}
diff --git a/parsers/manifest_parser.go b/parsers/manifest_parser.go
index 763dcbca..3c322f6e 100644
--- a/parsers/manifest_parser.go
+++ b/parsers/manifest_parser.go
@@ -27,13 +27,14 @@ import (
"gopkg.in/yaml.v2"
+ "path/filepath"
+
"github.com/apache/incubator-openwhisk-client-go/whisk"
"github.com/apache/incubator-openwhisk-wskdeploy/utils"
"github.com/apache/incubator-openwhisk-wskdeploy/wskderrors"
"github.com/apache/incubator-openwhisk-wskdeploy/wskenv"
"github.com/apache/incubator-openwhisk-wskdeploy/wski18n"
"github.com/apache/incubator-openwhisk-wskdeploy/wskprint"
- "path/filepath"
)
const (
@@ -141,7 +142,7 @@ func (dm *YAMLParser) composeAnnotations(annotations map[string]interface{}) whi
return listOfAnnotations
}
-func (dm *YAMLParser) ComposeDependenciesFromAllPackages(manifest *YAML, projectPath string, filePath string) (map[string]utils.DependencyRecord, error) {
+func (dm *YAMLParser) ComposeDependenciesFromAllPackages(manifest *YAML, projectPath string, filePath string, ma whisk.KeyValue) (map[string]utils.DependencyRecord, error) {
dependencies := make(map[string]utils.DependencyRecord)
packages := make(map[string]Package)
@@ -152,7 +153,7 @@ func (dm *YAMLParser) ComposeDependenciesFromAllPackages(manifest *YAML, project
}
for n, p := range packages {
- d, err := dm.ComposeDependencies(p, projectPath, filePath, n)
+ d, err := dm.ComposeDependencies(p, projectPath, filePath, n, ma)
if err == nil {
for k, v := range d {
dependencies[k] = v
@@ -164,7 +165,7 @@ func (dm *YAMLParser) ComposeDependenciesFromAllPackages(manifest *YAML, project
return dependencies, nil
}
-func (dm *YAMLParser) ComposeDependencies(pkg Package, projectPath string, filePath string, packageName string) (map[string]utils.DependencyRecord, error) {
+func (dm *YAMLParser) ComposeDependencies(pkg Package, projectPath string, filePath string, packageName string, ma whisk.KeyValue) (map[string]utils.DependencyRecord, error) {
depMap := make(map[string]utils.DependencyRecord)
for key, dependency := range pkg.Dependencies {
@@ -201,6 +202,11 @@ func (dm *YAMLParser) ComposeDependencies(pkg Package, projectPath string, fileP
annotations := dm.composeAnnotations(dependency.Annotations)
+ // add Managed Annotations if this is Managed Deployment
+ if utils.Flags.Managed {
+ annotations = append(annotations, ma)
+ }
+
packDir := path.Join(projectPath, strings.Title(YAML_KEY_PACKAGES))
depName := packageName + ":" + key
depMap[depName] = utils.NewDependencyRecord(packDir, packageName, location, version, inputs, annotations, isBinding)
diff --git a/parsers/manifest_parser_test.go b/parsers/manifest_parser_test.go
index bc6948eb..6c644922 100644
--- a/parsers/manifest_parser_test.go
+++ b/parsers/manifest_parser_test.go
@@ -21,11 +21,6 @@ package parsers
import (
"fmt"
- "github.com/apache/incubator-openwhisk-client-go/whisk"
- "github.com/apache/incubator-openwhisk-wskdeploy/utils"
- "github.com/apache/incubator-openwhisk-wskdeploy/wskderrors"
- "github.com/apache/incubator-openwhisk-wskdeploy/wskprint"
- "github.com/stretchr/testify/assert"
"io/ioutil"
"os"
"path/filepath"
@@ -33,6 +28,12 @@ import (
"strconv"
"strings"
"testing"
+
+ "github.com/apache/incubator-openwhisk-client-go/whisk"
+ "github.com/apache/incubator-openwhisk-wskdeploy/utils"
+ "github.com/apache/incubator-openwhisk-wskdeploy/wskderrors"
+ "github.com/apache/incubator-openwhisk-wskdeploy/wskprint"
+ "github.com/stretchr/testify/assert"
)
const (
@@ -1332,7 +1333,7 @@ func TestComposeDependencies(t *testing.T) {
file := "../tests/dat/manifest_data_compose_dependencies.yaml"
p, m, _ := testLoadParseManifest(t, file)
- depdList, err := p.ComposeDependenciesFromAllPackages(m, "/project_folder", m.Filepath)
+ depdList, err := p.ComposeDependenciesFromAllPackages(m, "/project_folder", m.Filepath, whisk.KeyValue{})
assert.Nil(t, err, fmt.Sprintf(TEST_ERROR_COMPOSE_DEPENDENCY_FAILURE, file))
assert.Equal(t, 3, len(depdList), "Failed to get rules")
diff --git a/parsers/yamlparser.go b/parsers/yamlparser.go
index c136dfed..c65896f3 100644
--- a/parsers/yamlparser.go
+++ b/parsers/yamlparser.go
@@ -416,6 +416,13 @@ func (yaml *YAML) ComposeParsersTrigger(wsktrg whisk.Trigger) *Trigger {
return trigger
}
+func (yaml *YAML) ComposeParsersDependency(binding whisk.Binding) *Dependency {
+ dependency := new(Dependency)
+ dependency.Location = "/" + binding.Namespace + "/" + binding.Name
+
+ return dependency
+}
+
func (yaml *YAML) ComposeParsersRule(wskrule whisk.Rule) *Rule {
rule := new(Rule)
rule.Name = wskrule.Name
diff --git a/utils/managedannotations.go b/utils/managedannotations.go
index fa0781ee..c0be08a2 100644
--- a/utils/managedannotations.go
+++ b/utils/managedannotations.go
@@ -21,8 +21,9 @@ import (
"crypto/sha1"
"encoding/json"
"fmt"
- "github.com/apache/incubator-openwhisk-client-go/whisk"
"os"
+
+ "github.com/apache/incubator-openwhisk-client-go/whisk"
)
/*
@@ -40,6 +41,7 @@ const (
NULL = "golang\000"
OW_PROJECT_NAME = "__OW_PROJECT_NAME"
OW_PROJECT_HASH = "__OW_PROJECT_HASH"
+ BINDING = "binding"
)
type ManagedAnnotation struct {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services