You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openserverless.apache.org by fr...@apache.org on 2024/08/06 13:40:33 UTC
(openserverless-operator) 01/24: first commit
This is an automated email from the ASF dual-hosted git repository.
franztt pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/openserverless-operator.git
commit d17d6564855fc3884d68caff7d4f007f24c14105
Author: Francesco Timperi Tiberi <fr...@gmail.com>
AuthorDate: Tue Jul 9 17:19:33 2024 +0000
first commit
---
DESIGN.md | 67 +
Dockerfile | 127 +
LICENSE | 218 +
OPERATOR.md | 247 +
README.md | 204 +
Taskfile.yml | 345 ++
TaskfileBuild.yml | 63 +
TaskfileDev.yml | 104 +
TaskfileEnterprise.yml | 71 +
TaskfileNuvEnterprise.yml | 91 +
TaskfileOlaris.yml | 29 +
TaskfileTest.yml | 197 +
actions/Taskfile.yml | 448 ++
actions/common/authorize.py | 162 +
actions/common/command_data.py | 50 +
actions/common/minio_util.py | 182 +
actions/common/util.py | 35 +
actions/content/__main__.py | 110 +
actions/content/common/minio_util.py | 182 +
actions/devel/download/__main__.py | 91 +
actions/devel/download/common/authorize.py | 162 +
actions/devel/download/common/command_data.py | 50 +
actions/devel/download/common/minio_util.py | 182 +
actions/devel/download/common/util.py | 35 +
actions/devel/download/flatdict.py | 506 ++
actions/devel/download/nuvolaris/config.py | 202 +
actions/devel/download/nuvolaris/couchdb_util.py | 164 +
actions/devel/ferretdb/__main__.py | 73 +
actions/devel/ferretdb/command/ferretbd.py | 75 +
actions/devel/ferretdb/common/authorize.py | 162 +
actions/devel/ferretdb/common/command_data.py | 50 +
actions/devel/ferretdb/common/minio_util.py | 182 +
actions/devel/ferretdb/common/util.py | 35 +
actions/devel/ferretdb/flatdict.py | 506 ++
actions/devel/ferretdb/nuvolaris/config.py | 202 +
actions/devel/ferretdb/nuvolaris/couchdb_util.py | 164 +
actions/devel/minio/__main__.py | 77 +
actions/devel/minio/command/minio.py | 170 +
actions/devel/minio/common/authorize.py | 162 +
actions/devel/minio/common/command_data.py | 50 +
actions/devel/minio/common/minio_util.py | 182 +
actions/devel/minio/common/util.py | 35 +
actions/devel/minio/flatdict.py | 506 ++
actions/devel/minio/nuvolaris/config.py | 202 +
actions/devel/minio/nuvolaris/couchdb_util.py | 164 +
actions/devel/psql/__main__.py | 74 +
actions/devel/psql/command/psql.py | 84 +
actions/devel/psql/common/authorize.py | 162 +
actions/devel/psql/common/command_data.py | 50 +
actions/devel/psql/common/minio_util.py | 182 +
actions/devel/psql/common/util.py | 35 +
actions/devel/psql/flatdict.py | 506 ++
actions/devel/psql/nuvolaris/config.py | 202 +
actions/devel/psql/nuvolaris/couchdb_util.py | 164 +
actions/devel/redis/__main__.py | 72 +
actions/devel/redis/command/redis.py | 55 +
actions/devel/redis/common/authorize.py | 162 +
actions/devel/redis/common/command_data.py | 50 +
actions/devel/redis/common/minio_util.py | 182 +
actions/devel/redis/common/util.py | 35 +
actions/devel/redis/flatdict.py | 506 ++
actions/devel/redis/nuvolaris/config.py | 202 +
actions/devel/redis/nuvolaris/couchdb_util.py | 164 +
actions/devel/upload/__main__.py | 115 +
actions/devel/upload/common/authorize.py | 162 +
actions/devel/upload/common/command_data.py | 50 +
actions/devel/upload/common/minio_util.py | 182 +
actions/devel/upload/common/util.py | 35 +
actions/devel/upload/flatdict.py | 506 ++
actions/devel/upload/nuvolaris/config.py | 202 +
actions/devel/upload/nuvolaris/couchdb_util.py | 164 +
actions/github/_playground.txt | 744 +++
actions/github/callback/__main__.py | 98 +
actions/github/common/config_exception.py | 19 +
actions/github/common/github_client.py | 96 +
actions/github/common/html_response_generator.py | 76 +
actions/github/common/kube_api_client.py | 143 +
actions/github/common/whisk_user_generator.py | 71 +
actions/github/webhook/__main__.py | 113 +
actions/login/__main__.py | 96 +
actions/login/flatdict.py | 506 ++
actions/system.yml | 126 +
clusters/aks.yml | 32 +
clusters/eks-cluster.yml | 40 +
clusters/eks.yml | 48 +
clusters/gke.yml | 40 +
clusters/juju-aws-overlay.yaml | 29 +
clusters/juju.yml | 25 +
clusters/k3s.yml | 40 +
clusters/kind.yml | 40 +
clusters/lks.yml | 51 +
clusters/microk8s.cf | 86 +
clusters/microk8s.yml | 95 +
clusters/openshift-aggregate.yaml | 41 +
clusters/openshift-install-config.yaml | 55 +
clusters/openshift-user.yaml | 40 +
clusters/openshift.yml | 63 +
clusters/talos-gcp.yml | 107 +
cron.sh | 19 +
dbinit.sh | 19 +
demos/devel/mdb_find.json | 1 +
demos/devel/mdb_insert.json | 18 +
demos/devel/minio_buckets.json | 3 +
demos/devel/minio_cp.json | 9 +
demos/devel/minio_ls.json | 6 +
demos/devel/minio_mv.json | 9 +
demos/devel/minio_rm.json | 7 +
demos/devel/psql_create1.json | 3 +
demos/devel/psql_create2.json | 3 +
demos/devel/psql_create3.json | 3 +
demos/devel/psql_desc.json | 3 +
demos/devel/psql_select.json | 3 +
demos/devel/redis_get.json | 3 +
demos/devel/redis_set.json | 3 +
demos/hellonode/.gitignore | 19 +
demos/hellonode/Dockerfile | 26 +
demos/hellonode/app.js | 30 +
demos/hellonode/deploy.yaml | 55 +
demos/hellonode/package-lock.json | 1018 ++++
demos/hellonode/package.json | 15 +
demos/operator/demo-crd.yaml | 71 +
demos/operator/demo-deployment.yaml | 37 +
demos/operator/demo-ns.yaml | 22 +
demos/operator/demo-obj.yaml | 26 +
demos/operator/demo1.py | 39 +
demos/operator/demo2.py | 61 +
demos/operator/demomock.py | 27 +
demos/operator/obj.yaml | 32 +
demos/operator/patch.yaml | 24 +
demos/operator/poetry.lock | 1179 +++++
demos/operator/pyproject.toml | 35 +
demos/operator/transcript.txt | 78 +
demos/velero/Taskfile.yml | 154 +
demos/velero/values.yaml | 538 ++
deploy/Taskfile.yml | 64 +
deploy/_deprecated/preload-runtime-go117.yaml | 29 +
deploy/_deprecated/preload-runtime-nodej14.yaml | 28 +
deploy/_deprecated/preload-runtime-python37.yaml | 29 +
deploy/_deprecated/standalone-kcf.yaml | 77 +
deploy/_deprecated/standalone-lb.yaml | 29 +
deploy/_deprecated/standalone-np.yaml | 58 +
deploy/_deprecated/standalone-set.yaml | 61 +
deploy/_deprecated/standalone-test-in-https.yaml | 47 +
deploy/acme-openshift/acme-rbac.yaml | 114 +
deploy/acme-openshift/acme-setup.yaml | 58 +
deploy/alert-manager/alertmanager-02-cm.yaml | 32 +
deploy/alert-manager/alertmanager-03-dep.yaml | 100 +
deploy/alert-manager/alertmanager-04-svc.yaml | 37 +
deploy/apisix-etcd/etcd-policy.yaml | 31 +
deploy/apisix-etcd/etcd-pvc.yaml | 32 +
deploy/apisix-etcd/etcd-secret.yaml | 28 +
deploy/apisix-etcd/etcd-sts.yaml | 160 +
deploy/apisix-etcd/etcd-svc.yaml | 64 +
deploy/apisix/apisix-cm.yaml | 248 +
deploy/apisix/apisix-dep.yaml | 83 +
deploy/apisix/apisix-svc.yaml | 60 +
deploy/cert-manager/cert-manager.yaml | 5521 ++++++++++++++++++++
deploy/content/index.html | 32 +
deploy/couchdb/couchdb-svc.yaml | 45 +
deploy/ferretdb/ferretdb-sts.yaml | 44 +
deploy/ferretdb/ferretdb-svc.yaml | 31 +
deploy/ingress-nginx/cloud-nginx-ingress.yaml | 666 +++
deploy/ingress-nginx/eks-nginx-ingress.yaml | 680 +++
deploy/ingress-nginx/kind-nginx-ingress.yaml | 690 +++
deploy/issuer/cluster-issuer.yaml | 36 +
deploy/kafka/kafka-000-pvc.yaml | 33 +
deploy/kafka/kafka-001-sts.yaml | 107 +
deploy/kafka/kafka-003-svc.yaml | 38 +
deploy/kvrocks/kvrocks-pvc.yaml | 28 +
deploy/kvrocks/kvrocks-sts.yaml | 62 +
deploy/kvrocks/kvrocks-svc.yaml | 31 +
deploy/minio/00-minio-pvc.yaml | 32 +
deploy/minio/01-minio-dep.yaml | 62 +
deploy/minio/02-minio-svc.yaml | 37 +
.../mongodb-auth-nuvolaris.yaml | 25 +
deploy/mongodb-operator-deploy/mongodb-auth.yaml | 25 +
deploy/mongodb-operator-deploy/mongodb-svc.yaml | 34 +
deploy/mongodb-operator-deploy/mongodb.yaml | 52 +
deploy/mongodb-operator/mongodb-crd.yaml | 451 ++
deploy/mongodb-operator/mongodb-operator.yaml | 92 +
deploy/mongodb-operator/mongodb-rbac.yaml | 123 +
deploy/mongodb-standalone/mongodb-auth.yaml | 25 +
deploy/mongodb-standalone/mongodb-cm.yaml | 34 +
deploy/mongodb-standalone/mongodb-pvc.yaml | 28 +
deploy/mongodb-standalone/mongodb-sts.yaml | 87 +
deploy/mongodb-standalone/mongodb-svc.yaml | 31 +
deploy/monitoring/kubestate-03-dep.yaml | 114 +
deploy/monitoring/kubestate-04-svc.yaml | 38 +
deploy/monitoring/prometheus-02-cm.yaml | 906 ++++
deploy/monitoring/prometheus-03-dep.yaml | 101 +
deploy/monitoring/prometheus-04-svc.yaml | 38 +
deploy/nginx-static/nginx-static-cm.yaml | 111 +
deploy/nginx-static/nginx-static-service.yaml | 33 +
deploy/nginx-static/nginx-static-sts.yaml | 66 +
deploy/ngrok | 0
deploy/nuvolaris-operator/operator-pod.yaml | 38 +
deploy/nuvolaris-operator/whisk.yaml | 79 +
deploy/nuvolaris-permissions/kubegres-crd.yaml | 3757 +++++++++++++
deploy/nuvolaris-permissions/nuvolaris-common.yaml | 36 +
.../nuvolaris-kubegres-common.yaml | 23 +
.../nuvolaris-kubegres-roles.yaml | 252 +
.../nuvolaris-monitoring-00-sa.yaml | 46 +
.../nuvolaris-monitoring-01-roles.yaml | 295 ++
.../nuvolaris-quota-common.yaml | 25 +
.../nuvolaris-quota-roles.yaml | 52 +
.../nuvolaris-wsku-common.yaml | 25 +
.../nuvolaris-wsku-roles.yaml | 57 +
.../openwhisk-core-roles.yaml | 57 +
.../nuvolaris-permissions/openwhisk-core-sa.yaml | 37 +
.../openwhisk-invoker-sa.yaml | 61 +
.../openwhisk-runtimes-cm.yaml | 401 ++
.../operator-clusterroles.yaml | 112 +
deploy/nuvolaris-permissions/operator-roles.yaml | 114 +
deploy/nuvolaris-permissions/whisk-crd.yaml | 827 +++
deploy/nuvolaris-permissions/whisk-user-crd.yaml | 230 +
deploy/nuvolaris-permissions/workflows-crd.yaml | 79 +
.../openwhisk-endpoint/operator-ingress-rbac.yaml | 52 +
deploy/openwhisk-enterprise/enterprise-np.yaml | 30 +
deploy/openwhisk-enterprise/enterprise-sts.yaml | 211 +
deploy/openwhisk-invoker/invoker-sts.yaml | 277 +
deploy/openwhisk-invoker/invoker-svc.yaml | 29 +
deploy/openwhisk-standalone/standalone-ip.yaml | 29 +
deploy/openwhisk-standalone/standalone-np.yaml | 44 +
deploy/openwhisk-standalone/standalone-sts.yaml | 240 +
deploy/postgres-backup/postgres-backup-conf.yaml | 60 +
deploy/postgres-backup/postgres-backup-sts.yaml | 90 +
deploy/postgres-operator-deploy/postgres-auth.yaml | 27 +
deploy/postgres-operator-deploy/postgres-conf.yaml | 60 +
deploy/postgres-operator-deploy/postgres.yaml | 45 +
deploy/postgres-operator/kubegres-operator.yaml | 142 +
deploy/qdrant/qdrant-cm.yaml | 47 +
deploy/qdrant/qdrant-sts.yaml | 141 +
deploy/qdrant/qdrant-svc.yaml | 80 +
deploy/quota/quota_checker.yaml | 53 +
deploy/redis/_redis-cm.yaml | 28 +
deploy/redis/redis-set.yaml | 56 +
deploy/redis/redis-svc.yaml | 31 +
deploy/runtimes/runtimes-preload.yaml | 46 +
deploy/s3ninja/s3ninja-set.yaml | 45 +
deploy/s3ninja/s3ninja-svc.yaml | 30 +
deploy/scheduler/cron.yaml | 43 +
deploy/test-cm/_index.html | 27 +
deploy/test-cm/pod.yaml | 35 +
deploy/test-cm/svc.yaml | 30 +
deploy/test-sts/nginx-set.yaml | 39 +
deploy/test-sts/nginx-svc.yaml | 30 +
deploy/test/_crd.yaml | 54 +
deploy/test/_obj.yaml | 25 +
deploy/test/_pv.yaml | 31 +
deploy/test/_pvc.yaml | 25 +
deploy/test/_test.json | 4 +
deploy/test/_testcm.yaml | 23 +
deploy/test/pod.yaml | 27 +
deploy/test/svc.yaml | 26 +
deploy/whisk-system/.gitkeep | 0
deploy/zookeeper/zookeeper-000-cm.yaml | 67 +
deploy/zookeeper/zookeeper-001-pvc.yaml | 47 +
deploy/zookeeper/zookeeper-003-sts.yaml | 87 +
deploy/zookeeper/zookeeper-004-svc.yaml | 35 +
detect.sh | 48 +
enterprise-util/couchdb/couchdb-nuvolaris-app.yaml | 42 +
.../couchdb/couchdb-nuvolaris-org-ingress.yaml | 42 +
enterprise-util/faas/.git-keep | 0
enterprise-util/faas/openwhisk-runtimes-cm.yaml | 289 +
enterprise-util/faas/whisk_nuvorg.yaml | 152 +
enterprise-util/kafka/Dockerfile | 22 +
enterprise-util/kafka/README.md | 27 +
enterprise-util/kafka/Taskfile.yml | 36 +
.../kafka/jmx/jmx_prometheus_javaagent-0.18.0.jar | Bin 0 -> 544974 bytes
.../kafka/jmx/prometheus-jmx-agent-config.yml | 141 +
enterprise-util/minio/minio-aks-ingress.yaml | 42 +
.../minio/minio-nuvolaris-org-ingress.yaml | 42 +
enterprise-util/minio/minio-saa1-ingress.yaml | 37 +
.../monitoring/prometheus-localhost.yaml | 37 +
.../prometheus-mistergpt-app-ingress.yaml | 37 +
.../prometheus-nuvolaris-app-ingress.yaml | 42 +
.../prometheus-nuvolaris-org-ingress.yaml | 42 +
enterprise/.git-keep | 0
init.src | 28 +
nuvolaris/__init__.py | 18 +
nuvolaris/actionexecutor.py | 396 ++
nuvolaris/apihost_util.py | 309 ++
nuvolaris/bucket.py | 31 +
nuvolaris/ceph_cos.py | 346 ++
nuvolaris/ceph_cos_ingress.py | 150 +
nuvolaris/ceph_cos_obc_data.py | 134 +
nuvolaris/ceph_cos_user_data.py | 137 +
nuvolaris/certmanager.py | 68 +
nuvolaris/config.py | 258 +
nuvolaris/couchdb.py | 277 +
nuvolaris/couchdb_util.py | 170 +
nuvolaris/cronjob.py | 99 +
nuvolaris/endpoint.py | 332 ++
nuvolaris/enterprise_util.py | 226 +
nuvolaris/ferretdb.py | 219 +
nuvolaris/files/README.md | 26 +
...vations_design_document_for_activations_db.json | 9 +
...uth_design_document_for_subjects_db_v2.0.0.json | 10 +
nuvolaris/files/filter_design_document.json | 6 +
...Cleanup_design_document_for_activations_db.json | 9 +
...hrottlings_design_document_for_subjects_db.json | 10 +
nuvolaris/files/runtimes-nodeonly.json | 46 +
nuvolaris/files/runtimes.json | 268 +
...document_for_activations_db_filters_v2.1.0.json | 10 +
...document_for_activations_db_filters_v2.1.1.json | 10 +
..._design_document_for_activations_db_v2.1.0.json | 10 +
...sks_design_document_for_entities_db_v2.1.0.json | 26 +
nuvolaris/ingress.py | 96 +
nuvolaris/ingress_data.py | 124 +
nuvolaris/invoker.py | 63 +
nuvolaris/ip_util.py | 108 +
nuvolaris/issuer.py | 99 +
nuvolaris/kafka.py | 81 +
nuvolaris/kopf_util.py | 171 +
nuvolaris/kube.py | 173 +
nuvolaris/kustomize.py | 349 ++
nuvolaris/kvrocks.py | 233 +
nuvolaris/main.py | 377 ++
nuvolaris/minio_deploy.py | 309 ++
nuvolaris/minio_ingress.py | 215 +
nuvolaris/minio_util.py | 169 +
nuvolaris/mongodb.py | 191 +
nuvolaris/mongodb_operator.py | 135 +
nuvolaris/mongodb_standalone.py | 73 +
nuvolaris/monitoring.py | 145 +
nuvolaris/nuvolaris_metadata.py | 87 +
nuvolaris/openwhisk.py | 43 +
nuvolaris/openwhisk_enterprise.py | 63 +
nuvolaris/openwhisk_standalone.py | 61 +
nuvolaris/operator_util.py | 135 +
nuvolaris/patcher.py | 177 +
nuvolaris/policies/minio_rw_policy_tpl.json | 19 +
nuvolaris/postgres_client.py | 145 +
nuvolaris/postgres_operator.py | 308 ++
nuvolaris/quota_checker.py | 215 +
nuvolaris/quota_checker_job.py | 105 +
nuvolaris/redis.py | 242 +
nuvolaris/redis_client.py | 130 +
nuvolaris/route_data.py | 87 +
nuvolaris/runtimes_preloader.py | 65 +
nuvolaris/runtimes_util.py | 55 +
nuvolaris/s3_bucket_policy.py | 98 +
nuvolaris/s3_client.py | 200 +
nuvolaris/storage_static.py | 267 +
nuvolaris/template.py | 84 +
nuvolaris/templates/00-minio-pvc.yaml | 32 +
nuvolaris/templates/01-minio-dep.yaml | 73 +
nuvolaris/templates/02-minio-svc.yaml | 37 +
.../affinity-tolerance-dep-core-attach.yaml | 62 +
.../affinity-tolerance-sts-core-attach.yaml | 62 +
.../affinity-tolerance-sts-invoker-attach.yaml | 54 +
nuvolaris/templates/alert-manager-02-cm.yaml | 113 +
nuvolaris/templates/cluster-issuer.yaml | 40 +
nuvolaris/templates/content.html | 27 +
nuvolaris/templates/cosi-obc-tpl.yaml | 29 +
nuvolaris/templates/cosi-user-tpl.yaml | 33 +
nuvolaris/templates/couchdb-init.yaml | 45 +
nuvolaris/templates/couchdb-set-tpl.yaml | 84 +
nuvolaris/templates/cron-init.yaml | 41 +
nuvolaris/templates/cronjob-attach.yaml | 31 +
nuvolaris/templates/dbname_pgpass_tpl.properties | 18 +
nuvolaris/templates/dep-attach.yaml | 47 +
nuvolaris/templates/enterprise-np.yaml | 30 +
nuvolaris/templates/enterprise-sts.yaml | 237 +
nuvolaris/templates/ferretdb-sts.yaml | 49 +
nuvolaris/templates/generic-ingress-tpl.yaml | 71 +
.../templates/generic-openshift-route-tpl.yaml | 45 +
nuvolaris/templates/index.html | 27 +
nuvolaris/templates/ingress.yaml | 32 +
nuvolaris/templates/invoker-sts.yaml | 308 ++
nuvolaris/templates/invoker-svc.yaml | 29 +
nuvolaris/templates/kafka-000-pvc.yaml | 33 +
nuvolaris/templates/kafka-001-sts.yaml | 109 +
nuvolaris/templates/kvrocks-cm.yaml | 936 ++++
nuvolaris/templates/kvrocks_manage_user_tpl.txt | 8 +
nuvolaris/templates/mongodb-auth-nuvolaris.yaml | 25 +
nuvolaris/templates/mongodb-auth.yaml | 25 +
nuvolaris/templates/mongodb-cm.yaml | 34 +
nuvolaris/templates/mongodb-config.yaml | 77 +
nuvolaris/templates/mongodb-sts.yaml | 90 +
nuvolaris/templates/mongodb_manage_user_tpl.js | 37 +
nuvolaris/templates/nginx-static-cm.yaml | 115 +
nuvolaris/templates/nginx-static-sts.yaml | 66 +
nuvolaris/templates/pgpass_tpl.properties | 18 +
nuvolaris/templates/postgres-backup-sts.yaml | 90 +
nuvolaris/templates/postgres.yaml | 76 +
.../postgres_manage_user_extension_tpl.sql | 30 +
nuvolaris/templates/postgres_manage_user_tpl.sql | 36 +
nuvolaris/templates/pv-create.yaml | 32 +
nuvolaris/templates/pvc-attach.yaml | 28 +
nuvolaris/templates/pvc-create.yaml | 31 +
nuvolaris/templates/quota-cronjob-attach.yaml | 34 +
nuvolaris/templates/redis-conf.yaml | 57 +
nuvolaris/templates/redis-set.yaml | 61 +
nuvolaris/templates/redis_manage_user_tpl.txt | 33 +
nuvolaris/templates/redis_quota_checker.lua | 21 +
nuvolaris/templates/redis_quota_checker.sh | 5 +
nuvolaris/templates/resources-sts-attach.yaml | 33 +
.../templates/runtimes-job-container-attach.yaml | 35 +
nuvolaris/templates/security-dep-attach.yaml | 36 +
nuvolaris/templates/security-global-attach.yaml | 28 +
nuvolaris/templates/security-set-attach.yaml | 36 +
nuvolaris/templates/set-attach.yaml | 43 +
nuvolaris/templates/standalone-sts.yaml | 258 +
nuvolaris/templates/subject.json | 11 +
nuvolaris/templates/test.json | 4 +
nuvolaris/templates/testcm.yaml | 23 +
nuvolaris/templates/traefik-middleware-tpl.yaml | 33 +
nuvolaris/templates/user_metadata.json | 28 +
nuvolaris/templates/whisk-system-manifest-tpl.yaml | 31 +
nuvolaris/templates/workflow-job.yaml | 42 +
nuvolaris/templates/zookeeper-001-pvc.yaml | 29 +
nuvolaris/templates/zookeeper-002-pvc.yaml | 29 +
nuvolaris/testutil.py | 303 ++
nuvolaris/time_util.py | 54 +
nuvolaris/user_config.py | 81 +
nuvolaris/user_handlers.py | 181 +
nuvolaris/user_metadata.py | 66 +
nuvolaris/user_patcher.py | 46 +
nuvolaris/userdb_util.py | 128 +
nuvolaris/util.py | 656 +++
nuvolaris/whisk_actions_deployer.py | 206 +
nuvolaris/whisk_system_util.py | 44 +
nuvolaris/workflows.py | 96 +
nuvolaris/zookeeper.py | 76 +
poetry.lock | 1678 ++++++
profile.ipy | 40 +
pyproject.toml | 49 +
quota.sh | 19 +
run.sh | 33 +
test_profile.ipy | 20 +
tests/aks/whisk.yaml | 120 +
tests/api.js | 32 +
tests/config_test.ipy | 62 +
tests/couchdb_test.ipy | 34 +
tests/cron_test.ipy | 25 +
tests/echo.js | 27 +
tests/eks/config_test.ipy | 23 +
tests/eks/whisk-full.yaml | 169 +
tests/eks/whisk-user.yaml | 44 +
tests/eks/whisk.yaml | 120 +
tests/generic/whisk-full.yaml | 153 +
tests/generic/whisk.yaml | 119 +
tests/gke/whisk.yaml | 120 +
tests/hello.js | 22 +
tests/hello.py | 24 +
tests/hi.py | 20 +
tests/invoker_test.ipy | 52 +
tests/k3s/config_test.ipy | 23 +
tests/k3s/ingress_test.ipy | 46 +
tests/k3s/minimal.yaml | 93 +
tests/k3s/traefik-middleware.yaml | 25 +
tests/k3s/whisk-demo-user.yaml | 47 +
tests/k3s/whisk-full-arm.yaml | 172 +
tests/k3s/whisk-full-faas.yaml | 172 +
tests/k3s/whisk-full-mysc2.yaml | 181 +
tests/k3s/whisk-full-saas.yaml | 174 +
tests/k3s/whisk-full.yaml | 174 +
tests/k3s/whisk-mysc2.yaml | 123 +
tests/k3s/whisk-saas.yaml | 174 +
tests/k3s/whisk-user.yaml | 47 +
tests/k3s/whisk.yaml | 120 +
tests/kafka_test.ipy | 54 +
tests/kind/action_executor_test.ipy | 48 +
tests/kind/config_test.ipy | 39 +
tests/kind/config_user_test.ipy | 24 +
tests/kind/couchdb_compactions_test.ipy | 55 +
tests/kind/couchdb_lib_test.ipy | 71 +
tests/kind/couchdb_util_test.ipy | 77 +
tests/kind/dbinit.ipy | 32 +
tests/kind/dbonly.yaml | 69 +
tests/kind/ferretdb_test.ipy | 42 +
tests/kind/minimal.yaml | 107 +
tests/kind/minio_static_test.ipy | 40 +
tests/kind/minio_test.ipy | 48 +
tests/kind/minio_util_test.ipy | 47 +
tests/kind/mongodb_op_test.ipy | 42 +
tests/kind/mongodb_std_test.ipy | 43 +
tests/kind/mongodb_user_test.ipy | 51 +
tests/kind/monitoring_test.ipy | 36 +
tests/kind/nuvolaris_subject_test.ipy | 70 +
tests/kind/openwhisk_enterprise_test.ipy | 89 +
tests/kind/postgres_op_test.ipy | 37 +
tests/kind/runtime_preloader_test.ipy | 35 +
tests/kind/userdb_util_test.ipy | 73 +
tests/kind/whisk-affinity.yaml | 137 +
tests/kind/whisk-full-affinity.yaml | 196 +
tests/kind/whisk-full-mf.yaml | 191 +
tests/kind/whisk-full-storage.yaml | 201 +
tests/kind/whisk-full.yaml | 206 +
tests/kind/whisk-gpt-user.yaml | 47 +
tests/kind/whisk-mf-demo.yaml | 140 +
tests/kind/whisk-minimal.yaml | 133 +
tests/kind/whisk-user.yaml | 51 +
tests/kind/whisk-user2.yaml | 51 +
tests/kind/whisk.yaml | 138 +
tests/kind/wskadmin.ipy | 43 +
tests/kube_test.py | 19 +
tests/kube_test.txt | 69 +
tests/kvrocks_test.ipy | 37 +
tests/kvrocks_user_test.ipy | 54 +
tests/lks/config_test.ipy | 24 +
tests/lks/whisk-full.yaml | 150 +
tests/lks/whisk.yaml | 118 +
tests/main_test.ipy | 36 +
tests/microk8s/config_test.ipy | 23 +
tests/microk8s/ingress_test.ipy | 46 +
tests/microk8s/monitoring-ingress.yaml | 24 +
tests/microk8s/whisk-full.yaml | 166 +
tests/microk8s/whisk-user.yaml | 47 +
tests/microk8s/whisk.yaml | 120 +
tests/minio-nuv.js | 47 +
tests/minio.js | 45 +
tests/mongo.js | 36 +
tests/mongo.yaml | 28 +
tests/mongodb_add_user_script.js | 30 +
tests/openshift/minimal.yaml | 103 +
tests/openshift/whisk-full.yaml | 138 +
tests/openshift/whisk-user.yaml | 47 +
tests/openshift/whisk.yaml | 119 +
tests/openwhisk_test.py | 64 +
tests/openwhisk_test.txt | 46 +
tests/patcher/openwhisk-runtimes-cm.yaml | 274 +
tests/patcher/whisk-full-baupanlabs.yaml | 134 +
tests/patcher/whisk-full-update.yaml | 137 +
tests/patcher/whisk-update.yaml | 102 +
tests/ping.js | 29 +
tests/postgres.js | 54 +
tests/quota_checker_test.ipy | 33 +
tests/redis.js | 51 +
tests/redis_test.ipy | 37 +
tests/redis_user_test.ipy | 55 +
tests/runtimes.json | 251 +
tests/runtimes/openwhisk-runtimes-cm.yaml | 317 ++
tests/volume_test.ipy | 71 +
tests/whisk-full.yaml | 204 +
tests/whisk-user.yaml | 43 +
tests/whisk.yaml | 119 +
tests/workflow-test.yaml | 36 +
tests/workflows_test.ipy | 55 +
tests/zookeeper_test.ipy | 43 +
tools/cli/wsk/wskadmin.py | 744 +++
tools/cli/wsk/wskprop.py | 82 +
tools/cli/wsk/wskutil.py | 106 +
whisk-system.sh | 83 +
wskadmin.sh | 48 +
546 files changed, 68246 insertions(+)
diff --git a/DESIGN.md b/DESIGN.md
new file mode 100644
index 0000000..423b5fc
--- /dev/null
+++ b/DESIGN.md
@@ -0,0 +1,67 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ ~
+-->
+# Design Document
+
+The operator is built with `kopf`. It is basically a server that will react to events coming from Kubernetes. The main evento to react to is the deployment of a Custom Resource Definition that configures OpenWhisk and all the related components.
+
+All the code is in the package `nuvolaris`. Events are handled in `main.py`.
+
+Each event handlers uses utility functions you can find the various service specific files like `openwhisk` or `couchdb`.
+
+## Development Workflow
+
+In generale the operator react to events and execute kustomizations.
+To develop,
+- you start and test writing a base YAML configuration in code.
+- you can then modify this base configuration in code using `kustomize` (wrapped in python)
+- finally you apply the configuration with `kubectl` (wapped in python)
+
+A good example of how to build a configuration is the following
+
+You start creating a basic definition under `deploy` for example `couchdb`. All the utils searches set of deployments under some folder under `deploy`.
+
+Then you can do this (obvious details omitted)
+
+```
+1 kust = kus.secretLiteral("couchdb-auth", user, pasw)
+2 kust += kus.patchTemplate("couchdb", "set-attach.yaml", data)
+3 spec = kus.kustom_list("couchdb", kust, templates=["couchdb-init.yaml"], data=data)
+4 kube.apply(spec)
+
+```
+
+In `1` you create a customization to create a secret.
+
+In `2` there is a customization to create a `patch` for it. There are a few utility functions for that in the `nuvolaris.kustomize` package. Some of them are actually templatized (have Template name) so the actual file is generated expanding a template. All the templates are under `nuvolaris/templates`. A templatized configuration uses `Jinja2`. When you use templates you also have to provide the `data` dictionary for the templates.
+
+3. Once you finisced you generate a configuration to be applied. Usually it is better to be a series of configuration, hence you invoke `kustom_list`. Note that kustom list, in addition to customize a configuration, can also add more configuration in the form of templates. Specificy the templates to use as `templates=templates` and do not forget to provide `data=data` to provide data
+
+4. Finally you apply the configuration with `kube.apply` - there are more utility functions running `kubectl` in a pythonic way.
+
+# Testing
+
+There are multiple level of testings
+
+- you first test algorithm with unit tests (`task utest` - I use a lot doctest)
+- you then test with integration tests without starting the operator (`task itest`) - tests uses ipython and assertions
+- you can run the operator locally without deploying it (with `task run`) and run task `deploy` to deploy a test CRD.
+- you can then deploy in local kind with `task build-and-load` without having to publish it, then execute tests (see the TaskfileTest.yml)
+- finally you test built the image and publish it with `task image-tag ; git push --tags`, wait it builds and test against real kubernetes, using the Taskfle
+
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..395ba23
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+FROM ubuntu:22.04
+ENV CONTROLLER_IMAGE=ghcr.io/nuvolaris/openwhisk-controller
+ENV CONTROLLER_TAG=3.1.0-mastrogpt.2402101445
+ENV INVOKER_IMAGE=ghcr.io/nuvolaris/openwhisk-invoker
+ENV INVOKER_TAG=3.1.0-mastrogpt.2402101445
+ARG OPERATOR_IMAGE_DEFAULT=ghcr.io/nuvolaris/openserverless-operator
+ARG OPERATOR_TAG_DEFAULT=1.3.0-beta.2309191654
+ENV OPERATOR_IMAGE=${OPERATOR_IMAGE_DEFAULT}
+ENV OPERATOR_TAG=${OPERATOR_TAG_DEFAULT}
+
+# configure dpkg && timezone
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+ENV TZ=Europe/London
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+# install Python
+RUN apt-get update && apt-get -y upgrade &&\
+ apt-get -y install apt-utils python3.10 python3.10-venv curl sudo telnet inetutils-ping zip unzip
+# Download Kubectl
+RUN KVER="v1.23.0" ;\
+ ARCH="$(dpkg --print-architecture)" ;\
+ KURL="https://dl.k8s.io/release/$KVER/bin/linux/$ARCH/kubectl" ;\
+ curl -sL $KURL -o /usr/bin/kubectl && chmod +x /usr/bin/kubectl
+RUN VER="v4.5.7" ;\
+ ARCH="$(dpkg --print-architecture)" ;\
+ URL="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F$VER/kustomize_${VER}_linux_${ARCH}.tar.gz" ;\
+ curl -sL "$URL" | tar xzvf - -C /usr/bin
+# Download WSK
+RUN WSK_VERSION=1.2.0 ;\
+ WSK_BASE=https://github.com/apache/openwhisk-cli/releases/download ;\
+ ARCH=$(dpkg --print-architecture) ;\
+ WSK_URL="$WSK_BASE/$WSK_VERSION/OpenWhisk_CLI-$WSK_VERSION-linux-$ARCH.tgz" ;\
+ curl -sL "$WSK_URL" | tar xzvf - -C /usr/bin/
+# Download MINIO client
+RUN rm -Rvf /tmp/minio-binaries ;\
+ mkdir /tmp/minio-binaries ;\
+ MINIO_BASE=https://dl.min.io/client/mc/release/linux ;\
+ ARCH=$(dpkg --print-architecture) ;\
+ MC_VER=RELEASE.2023-03-23T20-03-04Z ;\
+ MINIO_URL="$MINIO_BASE-$ARCH/archive/mc.${MC_VER}" ;\
+ curl -sL "$MINIO_URL" --create-dirs -o /tmp/minio-binaries/mc ;\
+ chmod +x /tmp/minio-binaries/mc ;\
+ mv /tmp/minio-binaries/mc /usr/bin/mc ;\
+ rm -Rvf /tmp/minio-binaries
+# Download and instal task
+RUN sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/bin
+# add nuvolaris user
+RUN groupadd --gid 1001 nuvolaris
+RUN useradd -m nuvolaris -s /bin/bash --uid 1001 --gid 1001 --groups root
+RUN echo "nuvolaris ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers
+
+#RUN useradd -m -s /bin/bash -g root -N nuvolaris && \
+# echo "nuvolaris ALL=(ALL:ALL) NOPASSWD: ALL" >>/etc/sudoers
+
+WORKDIR /home/nuvolaris
+# install the operator
+ADD nuvolaris/*.py /home/nuvolaris/nuvolaris/
+ADD nuvolaris/files /home/nuvolaris/nuvolaris/files
+ADD nuvolaris/templates /home/nuvolaris/nuvolaris/templates
+ADD nuvolaris/policies /home/nuvolaris/nuvolaris/policies
+ADD deploy/nuvolaris-operator /home/nuvolaris/deploy/nuvolaris-operator
+ADD deploy/nuvolaris-permissions /home/nuvolaris/deploy/nuvolaris-permissions
+ADD deploy/openwhisk-standalone /home/nuvolaris/deploy/openwhisk-standalone
+ADD deploy/openwhisk-endpoint /home/nuvolaris/deploy/openwhisk-endpoint
+ADD deploy/couchdb /home/nuvolaris/deploy/couchdb
+ADD deploy/redis /home/nuvolaris/deploy/redis
+ADD deploy/scheduler /home/nuvolaris/deploy/scheduler
+ADD deploy/mongodb-operator /home/nuvolaris/deploy/mongodb-operator
+ADD deploy/mongodb-operator-deploy /home/nuvolaris/deploy/mongodb-operator-deploy
+ADD deploy/mongodb-standalone /home/nuvolaris/deploy/mongodb-standalone
+ADD deploy/cert-manager /home/nuvolaris/deploy/cert-manager
+ADD deploy/ingress-nginx /home/nuvolaris/deploy/ingress-nginx
+ADD deploy/issuer /home/nuvolaris/deploy/issuer
+ADD deploy/minio /home/nuvolaris/deploy/minio
+ADD deploy/kafka /home/nuvolaris/deploy/kafka
+ADD deploy/zookeeper /home/nuvolaris/deploy/zookeeper
+ADD deploy/nginx-static /home/nuvolaris/deploy/nginx-static
+ADD deploy/content /home/nuvolaris/deploy/content
+ADD deploy/postgres-operator /home/nuvolaris/deploy/postgres-operator
+ADD deploy/postgres-operator-deploy /home/nuvolaris/deploy/postgres-operator-deploy
+ADD deploy/ferretdb /home/nuvolaris/deploy/ferretdb
+ADD deploy/runtimes /home/nuvolaris/deploy/runtimes
+ADD deploy/postgres-backup /home/nuvolaris/deploy/postgres-backup
+ADD run.sh dbinit.sh cron.sh pyproject.toml poetry.lock whisk-system.sh /home/nuvolaris/
+
+# prepares the required folders to deploy the whisk-system actions
+RUN mkdir /home/nuvolaris/deploy/whisk-system
+ADD actions /home/nuvolaris/actions
+
+# enterprise specific
+ADD deploy/openwhisk-enterprise /home/nuvolaris/deploy/openwhisk-enterprise
+ADD deploy/openwhisk-invoker /home/nuvolaris/deploy/openwhisk-invoker
+ADD deploy/monitoring /home/nuvolaris/deploy/monitoring
+ADD deploy/alert-manager /home/nuvolaris/deploy/alert-manager
+ADD deploy/quota /home/nuvolaris/deploy/quota
+ADD deploy/kvrocks /home/nuvolaris/deploy/kvrocks
+ADD quota.sh /home/nuvolaris/
+
+USER nuvolaris
+ENV PATH=/home/nuvolaris/.local/bin:/usr/local/bin:/usr/bin:/sbin:/bin
+RUN curl -sSL https://install.python-poetry.org | python3.10 -
+RUN poetry install
+USER root
+RUN chown -R nuvolaris /home/nuvolaris ;\
+ chgrp -R root /home/nuvolaris ;\
+ chmod -R 0775 /home/nuvolaris
+USER nuvolaris
+ENV HOME=/home/nuvolaris
+RUN ./whisk-system.sh
+RUN cd deploy && tar cvf ../deploy.tar *
+CMD ./run.sh
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..dc2c22e
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,218 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+========================================================================
+Apache License 2.0
+========================================================================
+
+This distribution bundles the following components, which are available under an Apache License 2.0 (https://opensource.org/licenses/Apache-2.0).
+Spray Caching 1.3.4 (io.spray:spray-caching_2.11:1.3.4 - http://spray.io/documentation/1.2.4/spray-caching/)
+ ConcurrentMapBackedCache.scala under common/scala/src/main/scala/org/apache/openwhisk/core/database/ contains implementation
+ from Spray's [[spray.caching.Cache]] and [[spray.caching.SimpleLruCache]] respectively.
+ License included at licenses/LICENSE-spray.txt, or https://github.com/spray/spray/blob/master/LICENSE
+ Copyright (C) 2011-2015 the spray project <http://spray.io>
+
+This product bundles the files gradlew and gradlew.bat from Gradle v5.5
+which are distributed under the Apache License, Version 2.0.
+For details see ./gradlew and ./gradlew.bat.
\ No newline at end of file
diff --git a/OPERATOR.md b/OPERATOR.md
new file mode 100644
index 0000000..6f41912
--- /dev/null
+++ b/OPERATOR.md
@@ -0,0 +1,247 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ ~
+-->
+# Apache OpenServerless Operator
+
+This is the Enterprise Kubernetes Operator of the [bit.ly/nuvolaris](nuvolaris project).
+
+## Developer notes
+
+We try to keep the enterprise version as closed as possible with the standalone version from the [bit.ly/nuvolaris](nuvolaris project).
+
+These features are exclusively provided by this enterprise version:
+
+- Kafka deployment including the corresponding Zookeeper
+- Openwhisk Invoker fully integrated with Zookeeper and Kafka
+- Openwhisk Controller fully integrated with Kafka
+- Possibility to deploy Openshisk Controller in lean mode
+
+## Publishing the operator
+Once the operator is ready, you can build and test it a against a kubernetes cluster.
+First, generate a new image tag with `task image-tag`.
+You can test locally using the kind cluster (provided by default by the development environment) with `ent:build-and-load`.
+
+
+To test it against other clusters, you need to publish it to the nuvolaris private repository on docker.hub
+
+To do this, add to your `.env` the following variables:
+
+
+```
+MY_CONTROLLER_IMAGE=<your-user>/<your-name>
+DOCKER_HUB_USER=<docker.hub user>
+DOCKER_HUB_TOKEN=<docker hub token>
+```
+
+To override the tag of the operator you have to generate a git tag: `task image-tag`. Note the tag is unique for the current hour (it embeds: YYMMDDhh).
+
+If you set those variables you can use
+- `task ent:docker-login` to log to the current docker.hub registry
+- `task ent:build-and-push` to build for one single architecture (faster but limited to your architecture)
+- `task ent:buildx-and-push` to build for all the architectues (slower, used by the GitHub action)
+- `task ent:docker-hub-authorize` to upload to the current kubernetes runtime a secret to give the possibility to download the operator enterprise image
+
+## Customization notes
+
+Nuvolaris operators it is normally deploy using a `whisk.yaml` configuration file which is applied via the installer. Typically the customization file contains something like
+
+```
+apiVersion: nuvolaris.org/v1
+kind: Whisk
+metadata:
+ name: controller
+ namespace: nuvolaris
+spec:
+ nuvolaris:
+ apihost: <ip address or hostname to be assigned to nuvolaris controller ingress>
+ components:
+ # start openwhisk controller
+ openwhisk: true
+ # start couchdb
+ couchdb: true
+ # start mongodb
+ mongodb: true
+ # start redis
+ redis: true
+ # start simple internal cron
+ cron: true
+ # tls enabled or not
+ tls: false
+ # minio enabled or not
+ minio: true
+ # start kafka
+ kafka: true
+ # start openwhisk invoker
+ invoker: true
+ # zookeeper enabled or not
+ zookeeper: true
+ openwhisk:
+ namespaces:
+ whisk-system: xxxx:yyyyyy
+ nuvolaris: ccccc:zzzzz
+ couchdb:
+ host: couchdb
+ port: 5984
+ volume-size: 10
+ admin:
+ user: <couch_db_admin_user>
+ password: <couch_db_admin_pwd>
+ controller:
+ user: <couch_db_controller_user>
+ password: <couch_db_controller_user>
+ mongodb:
+ host: mongodb
+ volume-size: 10
+ admin:
+ user: <mongodb_db_admin_user>
+ password: <mongodb_db_admin_pwd>
+ nuvolaris:
+ user: <mongodb_db_nuvolaris_user>
+ password: <mongodb_db_nuvolaris_pwd>
+ useOperator: False
+ scheduler:
+ schedule: "* * * * *"
+ tls:
+ acme-registered-email: xxxxx@youremailserver.com
+ acme-server-url: https://acme-staging-v02.api.letsencrypt.org/directory
+ minio:
+ volume-size: 2
+ nuvolaris:
+ root-user: <minio_admin_user>
+ root-password: <minio_admin_pwd>
+ configs:
+ limits:
+ actions:
+ sequence-maxLength: 50
+ invokes-perMinute: 999
+ invokes-concurrent: 250
+ triggers:
+ fires-perMinute: 999
+ time:
+ limit-min: "100ms"
+ limit-std: "1min"
+ limit-max: "5min"
+ memory:
+ limit-min: "128m"
+ limit-std: "256m"
+ limit-max: "512m"
+ controller:
+ javaOpts: "-Xmx2048M"
+ invoker:
+ javaOpts: "-Xmx2048M"
+ containerPool:
+ userMemory: "4096m"
+```
+
+## Default configs values
+If the provided `whisk.yaml` does not specify any dynamic configuration parameteres under `configs` item, the Enterprise operators defaults to these values:
+
+- `configs.limit.actions.sequence-maxLength=50`
+- `configs.limit.actions.invokes-perMinute=60`
+- `configs.limit.actions.invokes-concurrent=30`
+- `configs.limit.actions.triggers.fires-perMinute=60`
+- `configs.limits.time.limit-min=100ms`
+- `configs.limits.time.limit-std=1min`
+- `configs.limits.time.limit-max=5min`
+- `configs.limits.memory.limit-min=128m`
+- `configs.limits.memory.limit-std=256m`
+- `configs.limits.memory.limit-max=512m`
+- `configs.controller.javaOpts=-Xmx1024M`
+- `configs.controller.loggingLevel=INFO`
+- `configs.invoker.javaOpts=-Xmx1024M`
+- `configs.invoker.loggingLevel=INFO`
+- `configs.invoker.containerPool.userMemory=2048m`
+
+## Openwhisk Enterprise hot deployment
+
+The enterprise operator supports hot deployment for both Openwhisk Controller & Invoker, i.e it is possible to modify some specific part of the configuration inside the `whisk.yaml` file and apply it again. The operator will automatically stop and redeploy controller & invoker to take into account the new settings.
+
+```
+ configs:
+ limits:
+ actions:
+ sequence-maxLength: 50
+ invokes-perMinute: 999
+ invokes-concurrent: 250
+ triggers:
+ fires-perMinute: 999
+ time:
+ limit-min: "100ms"
+ limit-std: "1min"
+ limit-max: "5min"
+ memory:
+ limit-min: "128m"
+ limit-std: "256m"
+ limit-max: "512m"
+ controller:
+ javaOpts: "-Xmx2048M"
+ loggingLevel: "INFO"
+ invoker:
+ javaOpts: "-Xmx2048M"
+ loggingLevel: "INFO"
+ containerPool:
+ userMemory: "4096m"
+```
+
+As an example, using the above configuration and taking into account the standard 256m memory size limit per action, gives the possibility to execute around 16 action concurrently [`As a general rule consider it as the result of controller.javaOpts/256m`]. To increase the number of estimated action that can be executed modify for example `controller.javaOpts:"-Xmx8192M"` and redeploy the modified `whisk.yaml`.
+
+To apply the new customization execute
+
+```
+kubectl -n nuvolaris apply -f whisk.yaml
+```
+
+## Deploying the controller in lean-mode
+
+The enterprise operator supports also the deployment of the sole controller in lean mode, by simply applying a `whisk.yaml` containing
+
+```
+apiVersion: nuvolaris.org/v1
+kind: Whisk
+metadata:
+ name: controller
+ namespace: nuvolaris
+spec:
+ nuvolaris:
+ apihost: <ip address or hostname to be assigned to nuvolaris controller ingress>
+ components:
+ # start openwhisk controller
+ openwhisk: true
+ # start couchdb
+ couchdb: true
+ # start mongodb
+ mongodb: true
+ # start redis
+ redis: true
+ # start simple internal cron
+ cron: true
+ # tls enabled or not
+ tls: false
+ # minio enabled or not
+ minio: true
+ # start kafka
+ kafka: false
+ # start openwhisk invoker
+ invoker: false
+ # zookeeper enabled or not
+ zookeeper: false
+...
+```
+
+In lean mode it is recommended to set at least `configs.controller.javaOpts=-Xmx2048M`
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..65f7e97
--- /dev/null
+++ b/README.md
@@ -0,0 +1,204 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ ~
+-->
+# Apache OpenServerless Operator
+
+This is the Kubernetes Operator of the [bit.ly/nuvolaris](Apache OpenServerless project).
+
+If you are interested in developing it, please read the [design document](DESIGN.md). Please also read the [development environment document](https://github.com/nuvolaris/nuvolaris/blob/main/docs/DEVEL.md) to learn how to setup it.
+
+If you wonder what are the strange subfolders, here is an explanation:
+
+- `nuvolaris` are python modules, and implements the operator logic, coded in Python.
+
+So you generally build the `nuvolaris` operator as a docker image and publishi, build the `nuv` CLI and use the `olaris` tasks to install and manage it...
+
+## Developer notes
+
+The operator is built in Python with [kopf](https://kopf.readthedocs.io/en/stable/). You can find some [examples here](https://github.com/nolar/kopf/tree/main/examples).
+
+The operator uses under the hood [kustomize](https://kustomize.io/) and [kubectl](https://kubernetes.io/docs/reference/kubectl/) to interact with Kubernetes.
+
+Let's discuss how to:
+
+- Select the Kubernetes cluster to use
+- Develop interactively (cli and tests)
+- Run it without deploying
+- Test it in a local cluster without pushing an image to a registry
+- Push it to a public registry to use with external Kubernetes
+
+## Selecting the Kubernetes Clusters
+
+You have to run the operator against multiple clusters for test. So the operator has a provision to selection one to use and work with.
+
+You can use other clusters for testing and development. You should place all the configurations for the other clusters in the folder `clusters`, with extension `.kubeconfig`. See [below](#creating-a-new-cluster) for informations about this.
+
+If you use the development environment, using Docker and VSCode, you have available a Kind cluster ready for test. Use the task `task kind:config` to copy the configuration in the `clusters` folder if you plan to change cluster.
+
+In order to test the operator against different clusters, you can list all the cluster configurations available with `task` and then select one with `task N` where N is the number of the configuration in the list shown.
+
+Example:
+
+```
+$ task
+*** current: kind
+1 clusters/aks.kubeconfig
+2 clusters/eks.kubeconfig
+3 clusters/kind.kubeconfig
+4 clusters/microk8s.kubeconfig
+*** select with 'task #'
+$ task 2
+task: [2] task use N=2
+cluster: eks
+Kubeconfig user entry is using deprecated API version client.authentication.k8s.io/v1alpha1. Run 'aws eks update-kubeconfig' to update.
+NAME STATUS ROLES AGE VERSION
+ip-192-168-42-140.eu-central-1.compute.internal Ready <none> 49d v1.22.6-eks-7d68063
+```
+
+## Developing the operator interactively
+
+Start coding with `task cli`.
+
+You can then develop some code experimentally interacting with a python interpreter with the same libraries and some useful imports and configuration ready (most notably the autoreload).
+
+You can then execute some tests.
+
+- `task utest`: unit tests
+- `task itest`: integration tests
+
+This will run all the tests. You can select which test to run with `tast itest T=xxx` or `task utest T=xxx`. It will then execute all the tests whose file starts with `xxx`.
+
+## Developing the operator without deploying it
+
+You can then start the whole operator without having to deploy it. Start it with `task run`.
+
+Once you started the operarto pen another terminal and Use `task instance` to apply a crd instance depending on the current clusters.
+
+The crd instance used it is defined by the type of kubernetes custer you are using. The script './detect.sh' tries to detect it. The configuration used is under `tests/<detected>/whisk.yaml`
+
+If you have not changed the cluster the default cluster is 'kind' so it will apply the configuration `tests/kind/whisk.yaml`
+
+You can change the configuration passing the `WHISK` parameter (without the .yaml). So if you are running an `eks` cluster and you want to use your configuration `custom` the command `task instance WHISK=custom` will use `tests/eks/custom.yaml`
+
+Other useful tasks:
+
+- `task watch` starts watching deployed pods servics and nodes - useful to see what happens when you deploy
+- `task destroy` removes the instance but not the operator so you can recreate the cluster with `task instance` maybe with different parameters
+- `task clean` removes everything including the operator and pvc
+- If something get stuck and the cleanup does not complete `task defin` can help removing incomplete finalizers (you need to open another terminal to run it)
+
+## Testing the operator deploying it locally
+
+First and before all, you need a tag.
+
+Execute `task image-tag`. This command will delete all the other tag and generate a tag in the format `x.y.z-milestone.timestamp`. This tag is essential to identify your operator.
+
+When you are confident you can try the operator, in kind, use `task build-and-load`. This will build an instance and load it in the local kind. This step does not require you publish it to a public registry.
+
+You can then proceed deploying it with `task operator` to deploy it.
+
+You can then use `task instance` to deploy a crd instance, and finally `task actions` to test whisk running a few actions
+
+## Publishing the operator
+
+Once the operator is ready, you can build and test it a against a kubernetes cluster.
+
+First, generate a new image tag with `task image-tag`.
+
+You can test locally using the kind cluster (provided by default by the development environment) with `build-and-load`.
+
+To test it against other clusters, you need to publish it to a public repository.
+
+If you have push access to Nuvolaris repository just push the tag and it will trigger the build using the tag to tag the repository and it will be published in the nuvolaris package registry on GitHub.
+
+You can also publish an image to your own github repository for testing purposes. To do this, add to your `.env` the following variables:
+
+
+```
+MY_CONTROLLER_IMAGE=<your-user>/<your-name>
+GITHUB_USER=<your-user>
+GITHUB_TOKEN=<your-access-token>
+```
+
+You cannot override the tag of the operator you have to generate a git tag: `task image-tag`. Note the tag is unique for the current hour (it embeds: YYMMDDhh).
+
+If you set those variables you can use
+- `task b:docker-login` to log to the current docker registry
+- `task b:build-and-push` to build for one single architecture (faster but limited to your architecture)
+- `task buildx-and-push` to build for all the architectues (slower, used by the GitHub action)
+
+Remember that for GitHub Containter Registry you have to make public the image you are using to tlet it to be accessible by other Kubernetes
+
+### Running tests
+
+Once you have choosen the Kubernetes cluster and published the images on a public registry you can test it with `task dtest`, that will run a complete deployment in the current Kubernetes cluster.
+
+If you want to run all the tests (u, i, d) aginst the current cluster use `task test`
+
+Finally if you want to run the tests against all the configured kubernetes use `task all-kubes -- <target>`.
+
+This targes runs a group of tests against all the available clusters, so you can run all the tests against all the kubes with `task all-kubes -- test`
+
+# Reference
+
+- `use` (default task) lists all the clusters;
+- `1`, `2`...`9` selects the corresponding entry
+- `operator`: deploys the operator in the current cluster.
+- `instance`: deploy an instance of the configuration to build an actual cluster.
+- `destroy`: destroy the current deployment
+- `clean`: remove everything - if you get stuck use `defin`
+- `defin`: remove finalizer for the controller, useful if `clean` does not complete
+- `config`: once it is deployed, extracts the current configuration to use `wsk`
+- `actions`: deploy and run test actions
+- `utest` runs unit tests, filter with `T=<xxx>`
+- `itest` runs integration tests, filter with `T=<xxx>`
+- `iclean`: remote temp files generated by the tests
+- `dtest` deployment test
+- `hello`: runs a simple hello world test
+- `ping`: runs a ping test of redis
+- `all-kubes -- <target>`: execute the `<target>` against all the configured kubes
+
+Build targets
+
+- `b:buildx-and-push`: build and push the operator in multiple architectectures (slow)
+- `b:build-and-load`: build and load the operator in the kind cluster
+- `b:docker-login`: login into the github docker registry
+- `b:build-and-push`: build and push the operator in the current architecture
+
+Currently the following clusters, with respective suffix are supported (note not all the clusters are fully implemented, and you may need to edit and set manually variables - see the Taskfiles for reference).
+
+- Kubernetes Kind: `kind`
+- Amazon EKS: `eks`
+- Azure AKS: `aks`
+- Google GKE: `gke`
+- Ubuntu MicroK8s: `m8s`
+- Rancher K3S: `k3s`
+- RedHat OpenShift: `osh`
+
+See below for creating clusters and configurations. All the configuration for the available clusters are expected to be in `clusters/*.kubeconfig`
+
+Available commands for the cluster `xxx` (not always and not for all the cases):
+
+- `xxx:list`: list existing clusters
+- `xxx:create`: create a test cluster
+- `xxx:destroy`: destroy a test cluster
+- `xxx:config`: set the kubeconfig to the current cluster
+
+Note that `kind` is available in the development environment by defaut. You may want to use `task kind:config` to extract the configuration to be able to switch back to the local clusters.
+
diff --git a/Taskfile.yml b/Taskfile.yml
new file mode 100644
index 0000000..e6b17ef
--- /dev/null
+++ b/Taskfile.yml
@@ -0,0 +1,345 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: '3'
+
+vars:
+ G: ""
+ N: ""
+ T: ""
+ P: ""
+ BASETAG: 1.3.0-mastrogpt
+ KUBE:
+ sh: ./detect.sh
+ NS: "nuvolaris"
+ PREFIX: ""
+
+dotenv:
+ - .env
+
+includes:
+ d:
+ taskfile: TaskfileDev.yml
+ t:
+ taskfile: TaskfileTest.yml
+ b:
+ taskfile: TaskfileBuild.yml
+ o:
+ taskfile: TaskfileOlaris.yml
+ kind:
+ taskfile: clusters/kind.yml
+ dir: clusters
+ eks:
+ taskfile: clusters/eks.yml
+ dir: clusters
+ aks:
+ taskfile: clusters/aks.yml
+ dir: clusters
+ lks:
+ taskfile: clusters/lks.yml
+ dir: clusters
+ gke:
+ taskfile: clusters/gke.yml
+ dir: clusters
+ mik:
+ taskfile: clusters/microk8s.yml
+ dir: clusters
+ ent:
+ taskfile: TaskfileEnterprise.yml
+ k3s:
+ taskfile: clusters/k3s.yml
+ dir: clusters
+ osh:
+ taskfile: clusters/openshift.yml
+ dir: clusters
+ sys:
+ taskfile: actions/system.yml
+ dir: actions
+ talos:
+ taskfile: clusters/talos-gcp.yml
+ dir: clusters
+ nuvent:
+ taskfile: TaskfileNuvEnterprise.yml
+
+tasks:
+
+ default:
+ - task: use
+
+ setup:
+ deps:
+ - update-files-from-openwhisk
+ - ssh-key
+ - configure-env
+ cmds:
+ - mkdir -p ~/.kube ; touch ~/.kube/config
+ - poetry install
+ status:
+ - test -e $GOBIN/kopf
+
+ env: env
+
+ watch: watch kubectl -n {{.NS}} get nodes,pod,svc,pvc,ingress
+ watch-osh: watch kubectl -n {{.NS}} get nodes,pod,svc,pvc,route
+ watch-cert: watch kubectl -n {{.NS}} get ingress,ClusterIssuers,Certificates,CertificateRequests,Orders,Challenges
+ watch-pod: watch kubectl -n {{.NS}} get po,job --no-headers
+
+ logs: >
+ rm -f nuvolaris-operator.log ;
+ kubectl -n nuvolaris logs pod/nuvolaris-operator -f | tee nuvolaris-operator.log
+
+ cli:
+ - task: d:cli
+
+ run:
+ - task: sys:prepare
+ - task: t:permission
+ - task: d:run
+
+ irun:
+ - task: instance
+ - task: d:run
+
+ permission:
+ - task: t:permission
+
+ operator:
+ - task: t:operator
+
+ instance:
+ - task: t:instance
+
+ instance-wfx:
+ - task: d:instance-wfx
+
+ instance-and-log:
+ - task: d:instance
+ - kubectl -n nuvolaris logs pod/nuvolaris-operator -f
+
+ mongo:
+ - task: t:mongo
+
+ minio:
+ - task: t:minio
+
+ postgres:
+ - task: t:postgres
+
+ minimal:
+ #- task: permission
+ - task: operator
+ - task instance WHISK=minimal
+ - task: config
+ - task: hello
+
+ all:
+ - task: permission
+ - task: operator
+ - task: instance
+ - task: config
+ - task: hello
+ - task: redis
+ - task: mongo
+ - task: minio
+ - task: postgres
+
+ config:
+ - task: t:config
+
+ hello:
+ - task: t:hello
+
+ redis:
+ - task: t:redis
+
+ workflow:
+ - kubectl -n nuvolaris apply -f tests/workflow-test.yaml
+
+ defin:
+ - task: d:defin
+
+ image-tag:
+ - git tag -d $(git tag)
+ - git tag -f {{.P}}{{.BASETAG}}.$(date +%y%m%d%H%M)
+ - env PAGER= git tag
+
+ # configure env
+ configure-env:
+ cmds:
+ - |
+ if ! test -e .env
+ then echo "please copy .env.dist in .env and add the keys"
+ fi
+
+ # update configuration files from openwhisk source
+ update-files-from-openwhisk:
+ cmds:
+ - |
+ cp -v ../nuvolaris/nuvolaris-controller/openwhisk/ansible/files/*.json nuvolaris/files
+ cp -v ../nuvolaris/nuvolaris-controller/openwhisk/bin/wskadmin tools/cli/wsk/wskadmin.py
+ cp -v ../nuvolaris/nuvolaris-controller/openwhisk/tools/admin/*.py tools/cli/wsk
+ sources:
+ - ../nuvolaris/nuvolaris-controller/openwhisk/ansible/files/*.json
+ - ../nuvolaris/nuvolaris-controller/openwhisk/bin/wskadmin
+ - ../nuvolaris/nuvolaris-controller/openwhisk/tools/admin/*.py
+ generates:
+ - nuvolaris/files/*.json
+ - nuvolaris/tools/cli/wsk/*.py
+
+ # generate ssh keys
+ ssh-key:
+ cmds:
+ - test -f clusters/id_rsa || ssh-keygen -b 2048 -t rsa -f clusters/id_rsa -q -N ""
+ - ssh-keygen -y -f clusters/id_rsa >clusters/id_rsa.pub
+
+ clean:
+ cmds:
+ - cmd: kubectl -n nuvolaris delete wsku --all
+ ignore_error: true
+ - cmd: kubectl -n nuvolaris delete wsk/controller --grace-period=0 --timeout=5s
+ ignore_error: true
+ - cmd: task defin
+ ignore_error: true
+ - cmd: kubectl -n nuvolaris delete all --all --grace-period=0
+ ignore_error: true
+ - cmd: kubectl -n nuvolaris delete pvc --all --grace-period=0
+ ignore_error: true
+ - cmd: kubectl -n nuvolaris delete ing --all --grace-period=0
+ ignore_error: true
+ - cmd: kubectl -n nuvolaris delete kubegres --all
+ ignore_error: true
+ - cmd: kubectl -n nuvolaris delete cm/config --grace-period=0
+ ignore_error: true
+ - cmd: kubectl delete clusterissuers/letsencrypt-issuer
+ ignore_error: true
+
+ utest:
+ cmds:
+ - |
+ for test in nuvolaris/{{.T}}*.py
+ do echo "*** [{{.KUBE}}] $test"
+ poetry run python3 -m doctest $test {{.CLI_ARGS}}
+ done
+ silent: true
+
+ iclean: rm -f deploy/*/kustomization.yaml deploy/*/__* deploy/*/*_generated.yaml
+
+ itest:
+ cmds:
+ - task: iclean
+ - |
+ kubectl apply -f deploy/nuvolaris-permissions
+ rm -f _failed.txt
+ for test in tests/{{.T}}*.ipy tests/{{.KUBE}}/{{.T}}*.ipy
+ do
+ if test -e "$test"
+ then echo "*** [{{.KUBE}}] $test"
+ rm -f deploy/*/kustomization.yaml deploy/*/__*
+ if poetry run ipython $test {{.CLI_ARGS}}
+ then echo "OK: $test"
+ else echo "FAIL: $test"
+ echo $test >>_failed.txt
+ fi
+ fi
+ done
+ if test -e _failed.txt
+ then echo "*** FAILED TESTS:"
+ cat _failed.txt
+ fi
+ silent: true
+
+ dtest:
+ cmds:
+ - task: permission
+ - task: operator
+ - task: instance
+ - task: actions
+
+ actions:
+ - task: t:config
+ - task: t:hello
+ - task: t:redis
+ - task: t:echo
+ - task: t:api
+
+ test:
+ - task: clean
+ - task: utest
+ - task: itest
+ - task: dtest
+
+ debug:
+ - poetry run ipython profile create
+ - cp test_profile.ipy ~/.ipython/profile_default/startup/
+ - task: uitest
+
+ all-kubes:
+ cmds:
+ - |-
+ if test -z "{{.CLI_ARGS}}"
+ then echo 'use "task all-kubes -- <target> runs the target against all the available kubes'
+ else for cfg in clusters/*.kubeconfig
+ do /usr/games/cowsay -f duck $(basename $cfg .kubeconfig) 2>/dev/null
+ cp -v $cfg ~/.kube/config >/dev/null
+ task {{.CLI_ARGS}}
+ done
+ fi
+ silent: true
+
+ kube-test:
+ cmds:
+ - task dtest 2>/dev/null >/dev/null
+
+ use:
+ cmds:
+ - |-
+ if test -z "{{.N}}"
+ then echo "*** current: {{.KUBE}}"
+ ls clusters/*.kubeconfig | sort | awk '{ print NR, $0 }'
+ echo "*** select with 'task #'"
+ else CFG="$(ls -1 clusters/*.kubeconfig | tail +{{.N}} | head -1)"
+ cp $CFG ~/.kube/config
+ echo "cluster: $(./detect.sh)"
+ kubectl get nodes
+ fi
+ silent: true
+
+ build-and-load:
+ - task: b:build-and-load
+
+ build-and-push:
+ - task: b:build-and-push
+
+ buildx-and-push:
+ - task: b:buildx-and-push
+
+ docker-login:
+ - task: b:docker-login
+
+ shell:
+ - kubectl -n nuvolaris exec --stdin --tty {{.POD}} -- /bin/bash
+
+ 1: task use N=1
+ 2: task use N=2
+ 3: task use N=3
+ 4: task use N=4
+ 5: task use N=5
+ 6: task use N=6
+ 7: task use N=7
+ 8: task use N=8
+ 9: task use N=9
+
diff --git a/TaskfileBuild.yml b/TaskfileBuild.yml
new file mode 100644
index 0000000..74ff8a4
--- /dev/null
+++ b/TaskfileBuild.yml
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: '3'
+
+vars:
+ # taken from the tag
+ OPERATOR_TAG:
+ sh: git describe --tags --abbrev=0 2>/dev/null || git rev-parse --short HEAD
+ # taken from the Dockerfile - ovverdable with MY_OPERATOR_IMAGE
+ OPERATOR_IMAGE:
+ sh: awk -F= '/ARG OPERATOR_IMAGE_DEFAULT=/ { print $2 ; exit }' Dockerfile
+
+tasks:
+ image:
+ cmds:
+ - echo "Operator=${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}"
+ silent: true
+
+ docker-login: >
+ echo $GITHUB_TOKEN | docker login ghcr.io -u $GITHUB_USER --password-stdin
+
+ build:
+ - >
+ docker build .
+ -t ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}
+ --build-arg OPERATOR_IMAGE_DEFAULT=${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}
+ --build-arg OPERATOR_TAG_DEFAULT={{.OPERATOR_TAG}} --load
+
+ build-and-push:
+ - task: docker-login
+ - task: build
+ - docker push ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}
+
+ build-and-load:
+ - task: build
+ - >
+ kind load docker-image
+ ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}
+ --name=nuvolaris
+
+ buildx-and-push:
+ - >
+ docker buildx build
+ --platform linux/amd64,linux/arm64
+ -t ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}
+ --build-arg OPERATOR_IMAGE_DEFAULT=${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}
+ --build-arg OPERATOR_TAG_DEFAULT={{.OPERATOR_TAG}}
+ . --push
diff --git a/TaskfileDev.yml b/TaskfileDev.yml
new file mode 100644
index 0000000..5ea2695
--- /dev/null
+++ b/TaskfileDev.yml
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: '3'
+
+vars:
+ WHISK: '{{default "whisk" .WHISK}}'
+ CONFIG: "tests/{{.KUBE}}/{{.WHISK}}.yaml"
+ # taken from the tag
+ OPERATOR_TAG:
+ sh: git describe --tags --abbrev=0 2>/dev/null || git rev-parse --short HEAD
+ # taken from the Dockerfile - ovveridable with MY_OPERATOR_IMAGE
+ OPERATOR_IMAGE:
+ sh: awk -F= '/ARG OPERATOR_IMAGE_DEFAULT=/ { print $2 ; exit }' Dockerfile
+ # taken from the Dockerfile
+ CONTROLLER_TAG:
+ sh: awk -F= '/ENV CONTROLLER_TAG=/ { print $2 ; exit }' Dockerfile
+ CONTROLLER_IMAGE:
+ sh: awk -F= '/ENV CONTROLLER_IMAGE=/ { print $2 ; exit }' Dockerfile
+ # taken from the Dockerfile
+ INVOKER_TAG:
+ sh: awk -F= '/ENV INVOKER_TAG=/ { print $2 ; exit }' Dockerfile
+ INVOKER_IMAGE:
+ sh: awk -F= '/ENV INVOKER_IMAGE=/ { print $2 ; exit }' Dockerfile
+ APIKUBE:
+ sh: kubectl config view -o json | jq -r '.clusters[0].cluster.server'
+
+env:
+ APIHOST:
+ sh: echo {{.APIKUBE}} | sed -e 's|^.*//\(.*\):.*$|\1|'
+ APIHOST_OSH:
+ sh: echo {{.APIKUBE}} | sed -e 's|^.*//\(.*\):.*$|\1|' | sed -e 's/^api./nuvolaris.apps./'
+
+tasks:
+
+ env: env
+
+ cli:
+ cmds:
+ - env OPERATOR_IMAGE="${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}" poetry run ipython -i profile.ipy
+ env:
+ CONTROLLER_IMAGE: "{{.CONTROLLER_IMAGE}}"
+ CONTROLLER_TAG: "{{.CONTROLLER_TAG}}"
+ OPERATOR_TAG: "{{.OPERATOR_TAG}}"
+
+ run:
+ cmds:
+ # note that the .env is loaded after the env pameters
+ - env OPERATOR_IMAGE="${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}" ./run.sh
+ env:
+ CONTROLLER_IMAGE: "{{.CONTROLLER_IMAGE}}"
+ CONTROLLER_TAG: "{{.CONTROLLER_TAG}}"
+ OPERATOR_TAG: "{{.OPERATOR_TAG}}"
+ COUCHDB_SERVICE_HOST: "localhost"
+ MINIO_API_HOST: "localhost"
+ OW_CONTROLLER_HOST : "localhost"
+ OW_CONTROLLER_PORT : 3233
+ INVOKER_IMAGE: "{{.INVOKER_IMAGE}}"
+ INVOKER_TAG: "{{.INVOKER_TAG}}"
+
+ instance:
+ - envsubst <{{.CONFIG}} | kubectl -n nuvolaris apply -f -
+
+ show:
+ - echo "*** {{.CONFIG}}"
+ - envsubst <{{.CONFIG}}
+
+ destroy: kubectl -n nuvolaris delete wsk/controller
+
+ couchdb:
+ - rm -f deploy/*/kustomization.yaml deploy/*/__*
+ - task: permission
+ - kubectl apply -f deploy/couchdb
+
+ couchdb-forward:
+ - kubectl -n nuvolaris port-forward svc/couchdb 5985:5984
+
+ mongodb:
+ - kubectl apply -f deploy/mongodb-operator
+ - kubectl apply -f deploy/mongodb
+
+ defin: >
+ kubectl -n nuvolaris patch wsk/controller --type=merge --patch '{"metadata": {"finalizers":[] } }'
+
+ defin2: kubectl -n nuvolaris get wsk/controller -o yaml | grep -v Final | kubectl apply -f -
+
+ killpy: ps auwwx | grep python | awk '{print $2}' | xargs kill -9
+
+ enter: kubectl exec -ti nuvolaris-operator -- bash
+
diff --git a/TaskfileEnterprise.yml b/TaskfileEnterprise.yml
new file mode 100644
index 0000000..2a1ae28
--- /dev/null
+++ b/TaskfileEnterprise.yml
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: '3'
+
+vars:
+ # taken from the tag
+ OPERATOR_TAG:
+ sh: git describe --tags --abbrev=0 2>/dev/null || git rev-parse --short HEAD
+ # taken from the Dockerfile - ovverdable with MY_OPERATOR_IMAGE
+ OPERATOR_IMAGE:
+ sh: awk -F= '/ARG OPERATOR_IMAGE_DEFAULT=/ { print $2 ; exit }' Dockerfile
+
+tasks:
+ image:
+ cmds:
+ - echo "Operator=${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}"
+ silent: true
+
+ docker-login: >
+ echo $DOCKER_HUB_TOKEN | docker login registry.hub.docker.com -u $DOCKER_HUB_USER --password-stdin
+
+ build: >
+ docker build
+ -t ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}
+ --build-arg=OPERATOR_IMAGE_DEFAULT=${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}
+ --build-arg=OPERATOR_TAG_DEFAULT={{.OPERATOR_TAG}}
+ {{.CLI_ARGS}} .
+
+ build-and-push:
+ - task: docker-login
+ - task: build
+ - docker push ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}
+
+ build-and-load:
+ - task: build
+ - >
+ kind load docker-image
+ ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}
+ --name=nuvolaris
+
+ docker-setup:
+ - docker buildx create --use
+ - docker run -it --rm --privileged tonistiigi/binfmt --install all
+
+ buildx-and-push:
+ - task: docker-setup
+ - >
+ docker buildx build
+ --platform linux/amd64,linux/arm64
+ -t ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}:{{.OPERATOR_TAG}}
+ --build-arg OPERATOR_IMAGE_DEFAULT=${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}
+ --build-arg OPERATOR_TAG_DEFAULT={{.OPERATOR_TAG}}
+ . --push
+
+ docker-hub-authorize: >
+ kubectl create secret docker-registry regcred --docker-server=registry.hub.docker.com --docker-username=$DOCKER_HUB_USER --docker-password=$DOCKER_HUB_TOKEN
diff --git a/TaskfileNuvEnterprise.yml b/TaskfileNuvEnterprise.yml
new file mode 100644
index 0000000..56256da
--- /dev/null
+++ b/TaskfileNuvEnterprise.yml
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: '3'
+
+tasks:
+ devcluster:delete:
+ - nuv ent setup devcluster --uninstall
+
+ devcluster:
+ - nuv ent config enable --prometheus --slack --quota
+ - nuv ent config slack --apiurl=$SLACK_API_URL --channel=#nuvolaris-monitoring
+ - nuv ent config license nuvolaris --registry=registry.hub.docker.com --username=$DOCKER_HUB_USER --token=$DOCKER_HUB_TOKEN
+ - nuv ent config controller --javaopts=2 --replicas=1 --loglevel=INFO
+ - nuv ent config invoker --javaopts=2 --poolmemory=4 --timeoutsrun=5 --timeoutslogs=1 --replicas=1 --loglevel=INFO
+ - nuv ent config postgres --backup --schedule='30 * * * *' --replicas=2
+ - nuv ent config minio --s3 --console
+ - nuv ent setup devcluster
+
+ devcluster:full:
+ - nuv config reset
+ - nuv config enable --all
+ - task: devcluster
+
+ devcluster:minimal:
+ - nuv config reset
+ - nuv config disable --all
+ - task: devcluster
+
+ gc-deploy:
+ - nuv ent config controller --javaopts=-Xmx2048M
+ - nuv ent config invoker --javaopts=-Xmx2048M --poolmemory=8192m
+ - nuv ent config volumes --couchdb=30 --kafka=30 --database=50 --storage=50 --alerting=20
+ - nuv ent config storage --class=rook-ceph-block --provisioner=rook-ceph.rbd.csi.ceph.com
+ - nuv ent config enable --prometheus --slack
+ - nuv ent config slack --apiurl=$SLACK_API_URL --channel=#nuvolaris-monitoring
+ - nuv ent config license nuvolaris --registry=registry.hub.docker.com --username=$DOCKER_HUB_USER --token=$DOCKER_HUB_TOKEN
+ - nuv ent setup cluster
+
+ saas1:minimal:
+ - nuv config reset
+ - nuv config disable --all
+ - nuv config apihost 146.148.119.191
+ - task: gc-deploy
+
+ saas1:full:
+ - nuv config reset
+ - nuv config enable --all
+ - nuv config apihost 146.148.119.191
+ - task: gc-deploy
+
+ nuvolaris-app:minimal:
+ - nuv config reset
+ - nuv config disable --all
+ - nuv config apihost 35.242.186.170
+ - task: gc-deploy
+
+ nuvolaris-app:full:
+ - nuv config reset
+ - nuv config enable --all
+ - nuv config apihost 35.242.186.170
+ - task: gc-deploy
+
+ gc-cluster-kube:
+ - nuv ent saas admin gc-kubeconfig {{.N}}
+
+ gc-prepare:
+ - nuv ent saas gcloud vpc-create app-vpc
+ - nuv ent saas config gcloud --ent_name=app --ent_domain=nuvolaris.app --gc_region=europe-west2 --gc_zone=europe-west2-a
+
+ gc-cluster-delete:
+ - nuv ent saas admin gc-cluster-delete {{.N}}
+
+ gc-cluster-create:
+ - nuv ent saas admin gc-cluster-create {{.N}} --token=$GITHUB_OLARIS_CLONE_TOKEN
+
+
diff --git a/TaskfileOlaris.yml b/TaskfileOlaris.yml
new file mode 100644
index 0000000..6e80ad4
--- /dev/null
+++ b/TaskfileOlaris.yml
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version: '3'
+
+env:
+ DP: deploy/nuvolaris-permissions
+ OK: olaris/kubernetes
+
+tasks:
+ copy-yaml:
+ - cp -v $DP/nuvolaris-common.yaml $DP/openwhisk-runtimes-cm.yaml $OK/common
+ - cp -v $DP/openwhisk-core-roles.yaml $DP/operator-clusterroles.yaml $DP/operator-roles.yaml $OK/roles
+ - cp -v $DP/whisk-crd.yaml $DP/whisk-user-crd.yaml $DP/workflows-crd.yaml $OK/crds
+
\ No newline at end of file
diff --git a/TaskfileTest.yml b/TaskfileTest.yml
new file mode 100644
index 0000000..645b496
--- /dev/null
+++ b/TaskfileTest.yml
@@ -0,0 +1,197 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: '3'
+
+vars:
+ KUBE:
+ sh: ./detect.sh
+ # taken from the tag
+ OPERATOR_TAG:
+ sh: git describe --tags --abbrev=0 2>/dev/null || git rev-parse --short HEAD
+ # taken from the Dockerfile - ovverdable with MY_OPERATOR_IMAGE
+ OPERATOR_IMAGE:
+ sh: awk -F= '/ARG OPERATOR_IMAGE_DEFAULT=/ { print $2 ; exit }' Dockerfile
+
+ WHISK: '{{default "whisk" .WHISK}}'
+ CONFIG: "tests/{{.KUBE}}/{{.WHISK}}.yaml"
+ REDIS_URI: "redis://s0meP%40ass4@redis:6379"
+ REDIS_PASSWORD: s0meP@ass4
+ MINIO_HOST: "minio"
+ MINIO_PORT: 9000
+ MINIO_USER: "minioadmin"
+ MINIO_PWD : "minioadmin"
+ APIKUBE:
+ sh: kubectl config view -o json | jq -r '.clusters[0].cluster.server'
+
+ T: ""
+
+env:
+ APIHOST:
+ sh: echo {{.APIKUBE}} | sed -e 's|^.*//\(.*\):.*$|\1|'
+ APIHOST_OSH:
+ sh: echo {{.APIKUBE}} | sed -e 's|^.*//\(.*\):.*$|\1|' | sed -e 's/^api./nuvolaris.apps./'
+
+
+tasks:
+
+ kustomization:
+ cmds:
+ - |
+ cat <<__EOF__ >deploy/nuvolaris-operator/kustomization.yaml
+ apiVersion: kustomize.config.k8s.io/v1beta1
+ kind: Kustomization
+ images:
+ - name: ghcr.io/nuvolaris/nuvolaris-operator:latest
+ newName: ${MY_OPERATOR_IMAGE:-{{.OPERATOR_IMAGE}}}
+ newTag: {{.OPERATOR_TAG}}
+ resources:
+ - operator-pod.yaml
+ __EOF__
+
+ permission:
+ - kubectl apply -f deploy/nuvolaris-permissions
+
+ operator:
+ - task: kustomization
+ - kubectl apply -k deploy/nuvolaris-operator
+ - |
+ while ! kubectl -n nuvolaris wait --for=condition=ready pod/nuvolaris-operator
+ do echo still waiting...
+ done
+
+ destroy-operator:
+ - task: kustomization
+ - kubectl delete -k deploy/nuvolaris-operator
+
+ instance:
+ silent: false
+ cmds:
+ - kubectl config set-context --current --namespace nuvolaris
+ - >
+ cat tests/{{.KUBE}}/{{.WHISK}}.yaml | envsubst | kubectl apply -f -
+ - |
+ while ! kubectl -n nuvolaris wait --for=condition=ready pod/couchdb-0 2>/dev/null
+ do sleep 5 ; echo $((N++)) waiting couchdb...
+ done
+ - |
+ while ! kubectl -n nuvolaris wait --for=condition=complete job/couchdb-init 2>/dev/null
+ do sleep 5 ; echo $((N++)) waiting couchdb init...
+ done
+ - |
+ N=1
+ while ! kubectl -n nuvolaris wait --for=condition=ready pod/controller-0 2>/dev/null
+ do sleep 5 ; echo $((N++)) waiting controller...
+ done
+
+ destroy-instance:
+ - (sleep 5 ; task defin) &
+ - kubectl -n nuvolaris delete wsk/controller
+
+ destroy:
+ - task: destroy-instance
+ - task: destroy-operator
+
+ cleanup: kubectl -n nuvolaris delete pvc --all
+
+ config: |-
+ rm -f ~/.wskprops
+ while true
+ do APIHOST=$(kubectl -n nuvolaris get cm/config -o yaml | awk '/apihost:/ {print $2}')
+ if [[ `echo $APIHOST | grep pending` || -z "$APIHOST" ]];
+ then sleep 5 ; echo "$((N++)) apihost still pending..."
+ else break
+ fi
+ done
+ echo "*** $APIHOST ***"
+ AUTH=$(kubectl -n nuvolaris get wsk/controller -o jsonpath='{.spec.openwhisk.namespaces.nuvolaris}')
+ echo $AUTH
+ echo wsk property set --apihost $APIHOST --auth $AUTH
+ wsk property set --apihost $APIHOST --auth $AUTH
+ while ! wsk action list
+ do echo $(( N++)) "waiting for the load balancer to be ready..." ; sleep 10
+ done
+
+ hello:
+ - wsk -i action update hello tests/hello.js --web=true
+ - wsk -i action invoke hello -r | grep "hello"
+ - |
+ URL=$(wsk -i action get hello --url | tail +2)
+ curl -skL $URL | grep hello
+
+ redis:
+ cmds:
+ - wsk -i package update redis -p redis_url "{{.REDIS_NUV_URL}}" -p redis_prefix "{{.REDIS_NUV_PREFIX}}" -p password "{{.REDIS_PASSWORD}}"
+ - wsk -i action update redis/ping tests/ping.js
+ - wsk -i action invoke redis/ping -r | grep "PONG"
+ - wsk -i action update redis/redis tests/redis.js
+ - wsk -i action invoke redis/redis -r | grep "world"
+ vars:
+ REDIS_NUV_PREFIX:
+ sh: kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.redis_prefix}'
+ REDIS_NUV_URL:
+ sh: kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.redis_url}'
+
+ echo:
+ - wsk -i action update echo tests/echo.js -a provide-api-key true
+ - wsk -i action invoke echo -r | grep "__OW_API_KEY"
+ - wsk -i action invoke echo -r | grep "__OW_API_HOST"
+
+ api:
+ - wsk -i action update api tests/api.js -a provide-api-key true
+ - wsk -i action invoke api -r | grep '"api"'
+
+ mongo:
+ - |
+ MONGODB_URL=$(kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.mongodb_url}')
+ wsk -i package update mongo -p dburi "$MONGODB_URL"
+ - wsk -i action update mongo/mongo tests/mongo.js
+ - wsk -i action invoke mongo/mongo -r | grep "hello"
+
+ minio:
+ - wsk -i package update minio -p minio_host {{.MINIO_HOST}} -p minio_port {{.MINIO_PORT}} -p minio_user {{.MINIO_USER}} -p minio_pwd {{.MINIO_PWD}}
+ - wsk -i action update minio/minio tests/minio.js
+ - wsk -i action invoke minio/minio -r
+
+ mongo2:
+ - wsk -i project deploy --manifest tests/mongo.yaml
+
+ postgres:
+ - |
+ PG_URL=$(kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.postgres_url}')
+ wsk -i package update postgres -p dburi "$PG_URL"
+ - wsk -i action update postgres/postgres tests/postgres.js
+ - wsk -i action invoke postgres/postgres -r | grep "Nuvolaris Postgres is up and running!"
+
+ minio2:
+ silent: true
+ desc: minio test
+ cmds:
+ - |
+ MINIO_ACCESS_KEY=$(kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.minio_access_key}')
+ MINIO_SECRET_KEY=$(kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.minio_secret_key}')
+ MINIO_HOST=$(kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.minio_host}')
+ MINIO_PORT=$(kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.minio_port}')
+ MINIO_BUCKET_DATA=$(kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.minio_bucket_data}')
+ MINIO_BUCKET_WEB=$(kubectl -n nuvolaris get cm/config -o jsonpath='{.metadata.annotations.minio_bucket_static}')
+ wsk -i action update minio/minio-nuv tests/minio-nuv.js \
+ -p minio_access "$MINIO_ACCESS_KEY" \
+ -p minio_secret "$MINIO_SECRET_KEY" \
+ -p minio_host "$MINIO_HOST" \
+ -p minio_port "$MINIO_PORT" \
+ -p minio_data "$MINIO_BUCKET_DATA"
+ - wsk -i action invoke minio/minio-nuv -r
diff --git a/actions/Taskfile.yml b/actions/Taskfile.yml
new file mode 100644
index 0000000..1536972
--- /dev/null
+++ b/actions/Taskfile.yml
@@ -0,0 +1,448 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: '3'
+
+vars:
+ KUBE:
+ sh: ../detect.sh
+ WHISK: '{{default "whisk" .WHISK}}'
+ CONFIG: "../tests/{{.KUBE}}/{{.WHISK}}.yaml"
+ APIHOST:
+ sh: kubectl -n nuvolaris get cm/config -o yaml | awk '/apihost:/ {print $2}'
+ SYSTEM:
+ sh: kubectl -n nuvolaris get wsk/controller -o jsonpath='{.spec.openwhisk.namespaces.whisk-system}'
+ WSKSYS: wsk --apihost {{.APIHOST}} -u {{.SYSTEM}}
+ DEPLOY: ../deploy/whisk-system
+
+ CDB_USER:
+ sh: kubectl -n nuvolaris get wsk/controller -o jsonpath='{.spec.couchdb.admin.user}'
+
+ CDB_PASSWORD:
+ sh: kubectl -n nuvolaris get wsk/controller -o jsonpath='{.spec.couchdb.admin.password}'
+
+ NUV_MINIO_AUTH:
+ sh: kubectl -n nuvolaris get wsk/controller -o jsonpath='{.spec.minio.nuvolaris.password}'
+
+ CDB_HOST: couchdb
+
+ MINIO_HOST: minio.nuvolaris.svc.cluster.local
+ MINIO_PORT: 9000
+
+ #GIHTUB_SA_TOKEN:
+ # sh: kubectl -n nuvolaris get secrets/nuvolaris-github-secret -o jsonpath='{.data.token}'
+
+ #GIHTUB_SA_CRT:
+ # sh: kubectl -n nuvolaris get secrets/nuvolaris-github-secret -o jsonpath='{.data.ca\.crt}'
+
+ #GIHTUB_SA_NS:
+ # sh: kubectl -n nuvolaris get secrets/nuvolaris-github-secret -o jsonpath='{.data.namespace}'
+
+tasks:
+
+ list: "{{.WSKSYS}} action list"
+
+ action:get: "{{.WSKSYS}} action get {{.NAME}}"
+
+ activation:list: "{{.WSKSYS}} activation list"
+
+ activation:get: "{{.WSKSYS}} activation get {{.ID}}"
+
+ invoke: "{{.WSKSYS}} action invoke -r {{.CLI_ARGS}}"
+
+ config: wsk property set --apihost '{{.APIHOST}}' -u '{{.SYSTEM}}'
+
+ echo:
+ - zip -j {{.DEPLOY}}/echo.zip echo/*
+ - "{{.WSKSYS}} action update echo {{.DEPLOY}}/echo.zip --kind python:3"
+
+ login:prepare:
+ - |-
+ mkdir -p login/nuvolaris
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py login/nuvolaris
+ - |-
+ cd login
+ rm -f ../{{.DEPLOY}}/login.zip
+ zip -r ../{{.DEPLOY}}/login.zip *
+
+ login:deploy:
+ - >
+ {{.WSKSYS}} package update nuv
+ -p couchdb_user "{{.CDB_USER}}"
+ -p couchdb_password "{{.CDB_PASSWORD}}"
+ -p couchdb_host "{{.CDB_HOST}}"
+ - >
+ {{.WSKSYS}} action update nuv/login
+ {{.DEPLOY}}/login.zip --kind python:3 --web true
+ - >
+ {{.WSKSYS}} action get nuv/login --url
+
+ content:prepare:
+ - |-
+ mkdir -p content/common
+ cp ./common/minio_util.py content/common
+ - |-
+ cd content
+ rm -f ../{{.DEPLOY}}/content.zip
+ zip -r ../{{.DEPLOY}}/content.zip *
+
+ content:deploy:
+ - task: content:prepare
+ - >
+ {{.WSKSYS}} package update nuv
+ -p minio_host {{.MINIO_HOST}}
+ -p minio_port {{.MINIO_PORT}}
+ - >
+ {{.WSKSYS}} action update nuv/content
+ {{.DEPLOY}}/content.zip --kind python:3 --web true
+ - >
+ {{.WSKSYS}} action get nuv/content --url
+
+ devel:redis:prepare:
+ - |-
+ mkdir -p devel/redis/nuvolaris
+ mkdir -p devel/redis/common
+ cp ./common/*.py devel/redis/common
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py devel/redis/nuvolaris
+ - |-
+ cd devel/redis
+ rm -f ../../{{.DEPLOY}}/redis.zip
+ zip -r ../../{{.DEPLOY}}/redis.zip *
+
+ devel:redis:deploy:
+ - task: devel:redis:prepare
+ - >
+ {{.WSKSYS}} package update nuv
+ -p couchdb_user "{{.CDB_USER}}"
+ -p couchdb_password "{{.CDB_PASSWORD}}"
+ -p couchdb_host "{{.CDB_HOST}}"
+ - >
+ {{.WSKSYS}} action update nuv/redis
+ {{.DEPLOY}}/redis.zip --kind python:3 --web raw
+ - >
+ {{.WSKSYS}} action get nuv/redis --url
+
+ devel:psql:prepare:
+ - |-
+ mkdir -p devel/psql/nuvolaris
+ mkdir -p devel/psql/common
+ cp ./common/*.py devel/psql/common
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py devel/psql/nuvolaris
+ - |-
+ cd devel/psql
+ rm -f ../../{{.DEPLOY}}/psql.zip
+ zip -r ../../{{.DEPLOY}}/psql.zip *
+
+ devel:psql:deploy:
+ - task: devel:psql:prepare
+ - >
+ {{.WSKSYS}} package update nuv
+ -p couchdb_user "{{.CDB_USER}}"
+ -p couchdb_password "{{.CDB_PASSWORD}}"
+ -p couchdb_host "{{.CDB_HOST}}"
+ - >
+ {{.WSKSYS}} action update nuv/psql
+ {{.DEPLOY}}/psql.zip --kind python:3 --web raw
+ - >
+ {{.WSKSYS}} action get nuv/psql --url
+
+ devel:minio:prepare:
+ - |-
+ mkdir -p devel/minio/nuvolaris
+ mkdir -p devel/minio/common
+ cp ./common/*.py devel/minio/common
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py devel/minio/nuvolaris
+ - |-
+ cd devel/minio
+ rm -f ../../{{.DEPLOY}}/minio.zip
+ zip -r ../../{{.DEPLOY}}/minio.zip *
+
+ devel:minio:deploy:
+ - task: devel:minio:prepare
+ - >
+ {{.WSKSYS}} package update nuv
+ -p couchdb_user "{{.CDB_USER}}"
+ -p couchdb_password "{{.CDB_PASSWORD}}"
+ -p couchdb_host "{{.CDB_HOST}}"
+ - >
+ {{.WSKSYS}} action update nuv/minio
+ {{.DEPLOY}}/minio.zip --kind python:3 --web raw
+ - >
+ {{.WSKSYS}} action get nuv/minio --url
+
+ devel:upload:prepare:
+ - |-
+ mkdir -p devel/upload/nuvolaris
+ mkdir -p devel/upload/common
+ cp ./common/*.py devel/upload/common
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py devel/upload/nuvolaris
+ - |-
+ cd devel/upload
+ rm -f ../../{{.DEPLOY}}/devel_upload.zip
+ zip -r ../../{{.DEPLOY}}/devel_upload.zip *
+
+ devel:upload:deploy:
+ - task: devel:upload:prepare
+ - >
+ {{.WSKSYS}} package update nuv
+ -p couchdb_user "{{.CDB_USER}}"
+ -p couchdb_password "{{.CDB_PASSWORD}}"
+ -p couchdb_host "{{.CDB_HOST}}"
+ - >
+ {{.WSKSYS}} action update nuv/devel_upload
+ {{.DEPLOY}}/devel_upload.zip --kind python:3 --web raw
+ - >
+ {{.WSKSYS}} action get nuv/devel_upload --url
+
+ devel:ferretdb:prepare:
+ - |-
+ mkdir -p devel/ferretdb/nuvolaris
+ mkdir -p devel/ferretdb/common
+ cp ./common/*.py devel/ferretdb/common
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py devel/ferretdb/nuvolaris
+ - |-
+ cd devel/ferretdb
+ rm -f ../../{{.DEPLOY}}/ferretdb.zip
+ zip -r ../../{{.DEPLOY}}/ferretdb.zip *
+
+ devel:ferretdb:deploy:
+ - task: devel:ferretdb:prepare
+ - >
+ {{.WSKSYS}} package update nuv
+ -p couchdb_user "{{.CDB_USER}}"
+ -p couchdb_password "{{.CDB_PASSWORD}}"
+ -p couchdb_host "{{.CDB_HOST}}"
+ - >
+ {{.WSKSYS}} action update nuv/ferretdb
+ {{.DEPLOY}}/ferretdb.zip --kind python:3 --web raw
+ - >
+ {{.WSKSYS}} action get nuv/ferretdb --url
+
+ devel:download:prepare:
+ - |-
+ mkdir -p devel/download/nuvolaris
+ mkdir -p devel/download/common
+ cp ./common/*.py devel/download/common
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py devel/download/nuvolaris
+ - |-
+ cd devel/download
+ rm -f ../../{{.DEPLOY}}/devel_download.zip
+ zip -r ../../{{.DEPLOY}}/devel_download.zip *
+
+ devel:download:deploy:
+ - task: devel:download:prepare
+ - >
+ {{.WSKSYS}} package update nuv
+ -p couchdb_user "{{.CDB_USER}}"
+ -p couchdb_password "{{.CDB_PASSWORD}}"
+ -p couchdb_host "{{.CDB_HOST}}"
+ - >
+ {{.WSKSYS}} action update nuv/devel_download
+ {{.DEPLOY}}/devel_download.zip --kind python:3 --web raw
+ - >
+ {{.WSKSYS}} action get nuv/devel_download --url
+
+ login:webtest:
+ - >
+ curl -X POST {{.APIHOST}}/api/v1/web/whisk-system/nuv/login -H "Content-Type: application/json" -d '{"login": "{{.USERNAME}}", "password": "{{.PASSWORD}}"}'
+
+ login:webproxy:
+ - >
+ curl -X POST http://localhost/api/v1/web/whisk-system/nuv/login -H "Content-Type: application/json" -d '{"login": "{{.USERNAME}}", "password": "{{.PASSWORD}}"}'
+
+ login:k3sproxy:
+ - >
+ curl -X POST https://gptuser.api.k3s.nuvtest.net/api/v1/web/whisk-system/nuv/login -H "Content-Type: application/json" -d '{"login": "{{.USERNAME}}", "password": "{{.PASSWORD}}"}'
+
+ content:webtest:
+ - >
+ curl -X PUT -T ../nuvolaris/templates/index.html -H "minioauth: {{.NUV_MINIO_AUTH}}" {{.APIHOST}}/api/v1/web/whisk-system/nuv/content/nuvolaris/index2.html
+
+ content:webtest2:
+ - >
+ curl -X PUT -T ../nuvolaris/templates/content.html -H "minioauth: {{.NUV_MINIO_AUTH}}" {{.APIHOST}}/api/v1/web/whisk-system/nuv/content/nuvolaris/tidy/content.html
+
+ content:yaml:
+ - >
+ curl -X PUT -T ../nuvolaris/templates/couchdb-init.yaml -H "minioauth: {{.NUV_MINIO_AUTH}}" {{.APIHOST}}/api/v1/web/whisk-system/nuv/content/nuvolaris/yaml/couchdb-init.yaml
+
+ content-delete:yaml:
+ - >
+ curl -X DELETE -H "minioauth: {{.NUV_MINIO_AUTH}}" {{.APIHOST}}/api/v1/web/whisk-system/nuv/content/nuvolaris/yaml/couchdb-init.yaml
+
+ content:chess:
+ - >
+ curl -X PUT -T ../nuvolaris/templates/content.html -H "minioauth: {{.NUV_MINIO_AUTH}}" {{.APIHOST}}/api/v1/web/whisk-system/nuv/content/nuvolaris/chess/index.html
+
+ devel:redis:set:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/redis -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/redis_set.json
+
+ devel:redis:get:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/redis -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/redis_get.json
+
+ devel:psql:select:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/psql -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/psql_select.json
+
+ devel:psql:desc:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/psql -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/psql_desc.json
+
+ devel:psql:create:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/psql -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/psql_create1.json
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/psql -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/psql_create2.json
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/psql -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/psql_create3.json
+
+ devel:minio:ls:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/minio -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/minio_buckets.json
+
+ devel:minio:lsb:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/minio -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/minio_ls.json
+
+ devel:minio:rm:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/minio -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/minio_rm.json
+
+ devel:minio:mv:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/minio -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/minio_mv.json
+
+ devel:minio:cp:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/minio -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/minio_cp.json
+
+ devel:upload:webtest:
+ - >
+ curl -X PUT -T ../nuvolaris/templates/content.html -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" {{.APIHOST}}/api/v1/web/whisk-system/nuv/devel_upload/nuvolaris-web/test/index.html
+
+ devel:upload:data:
+ - >
+ curl -X PUT -T ../nuvolaris/templates/content.html -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" {{.APIHOST}}/api/v1/web/whisk-system/nuv/devel_upload/nuvolaris-data/test/index.html
+
+ devel:ferretdb:insert:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/ferretdb -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/mdb_insert.json
+
+ devel:ferretdb:find:
+ - >
+ curl {{.APIHOST}}/api/v1/web/whisk-system/nuv/ferretdb -H "x-impersonate-auth: Y2JkNjgwNzUtZGFjMi00NzVlLThjMDctZDYyYTMwYzdlNjgzOjEyM3pPM3haQ0xyTU42djJCS0sxZFhZRnBYbFBrY2NPRnFtMTJDZEFzTWdSVTRWck5aOWx5R1ZDR3VNREdJd1A=" -H "Content-Type: application/json" -d@../demos/devel/mdb_find.json
+
+
+
+ login:all:
+ - task: login:prepare
+ - task: login:deploy
+ - task: login:webtest
+
+ content:all:
+ - task: content:prepare
+ - task: content:deploy
+
+ # prepares all the system related actions
+ prepare:
+ - task: login:prepare
+ - task: content:prepare
+
+ deploy:
+ - task: login:deploy
+ - task: content:deploy
+
+ all:
+ - task: prepare
+ - task: deploy
+
+ deploy:devel:
+ - task: devel:redis:deploy
+ - task: devel:minio:deploy
+ - task: devel:psql:deploy
+ - task: devel:upload:deploy
+ - task: devel:ferretdb:deploy
+ - task: devel:download:deploy
+
+ github:echo:
+ silent: true
+ cmds:
+ - echo "{{.GIHTUB_SA_TOKEN}}"
+ - echo "{{.GIHTUB_SA_NS}}"
+ - echo "{{.GIHTUB_SA_CRT}}"
+
+ github:package:deploy:
+ - >
+ {{.WSKSYS}} package update github
+ -p couchdb_user "{{.CDB_USER}}"
+ -p couchdb_passwo986149f29564be9d0543a11d6657bca63deb7ff5rd "{{.CDB_PASSWORD}}"
+ -p couchdb_host "{{.CDB_HOST}}"
+ -p sa_token "{{.GIHTUB_SA_TOKEN}}"
+ -p sa_crt "{{.GIHTUB_SA_CRT}}"
+ -p sa_ns "{{.GIHTUB_SA_NS}}"
+ -p client_id "Iv1.d2bc07e358e18a70"
+ -p client_secret "986149f29564be9d0543a11d6657bca63deb7ff5"
+ -p kube_host "10.43.0.1"
+ -p kube_port "443"
+ -p apihost "api.arm.nuvtest.net"
+
+
+ github:callback:prepare:
+ - |-
+ mkdir -p github/callback/nuvolaris
+ mkdir -p github/callback/common
+ cp github/common/*.py github/callback/common
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py github/callback/nuvolaris
+ - |-
+ cd github/callback
+ rm -f ../../{{.DEPLOY}}/callback.zip
+ zip -r ../../{{.DEPLOY}}/callback.zip *
+
+ github:callback:deploy:
+ - task: github:callback:prepare
+ - >
+ {{.WSKSYS}} action update github/callback
+ {{.DEPLOY}}/callback.zip --kind python:3 --web true
+ - >
+ {{.WSKSYS}} action get github/callback --url
+
+ github:webhook:prepare:
+ - |-
+ mkdir -p github/webhook/nuvolaris
+ mkdir -p github/webhook/common
+ cp github/common/*.py github/webhook/common
+ cp ../nuvolaris/config.py ../nuvolaris/couchdb_util.py github/webhook/nuvolaris
+ - |-
+ cd github/webhook
+ rm -f ../../{{.DEPLOY}}/webhook.zip
+ zip -r ../../{{.DEPLOY}}/webhook.zip *
+
+ github:webhook:deploy:
+ - task: github:webhook:prepare
+ - >
+ {{.WSKSYS}} action update github/webhook
+ {{.DEPLOY}}/webhook.zip --kind python:3 --web raw
+ - >
+ {{.WSKSYS}} action get github/webhook --url
+
+ deploy:github:
+ - task: github:package:deploy
+ - task: github:callback:deploy
+ - task: github:webhook:deploy
+
diff --git a/actions/common/authorize.py b/actions/common/authorize.py
new file mode 100644
index 0000000..f7d716f
--- /dev/null
+++ b/actions/common/authorize.py
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from base64 import b64decode, b64encode
+from urllib.parse import quote, unquote
+
+import nuvolaris.config as cfg
+import nuvolaris.couchdb_util as cu
+import json
+import common.util as ut
+
+USER_META_DBN = "users_metadata"
+SUBJECT_META_DBN = "subjects"
+
+class DecodeError(Exception):
+ pass
+
+class EncodeError(Exception):
+ pass
+
+class AuthorizationError(Exception):
+ pass
+
+class Authorize():
+
+ def __init__(self, cdb_host, cdb_user, cdb_pwd):
+ cfg.clean()
+ cfg.put("couchdb.host", cdb_host)
+ cfg.put("couchdb.admin.user", cdb_user)
+ cfg.put("couchdb.admin.password", cdb_pwd)
+ self._db = cu.CouchDB()
+
+ def encode(self,username, password):
+ """Returns an HTTP basic authentication encrypted string given a valid
+ username and password.
+ """
+ if ':' in username:
+ raise EncodeError
+
+ username_password = f'{quote(username)}:{quote(password)}'
+ return f'Basic {b64encode(username_password.encode()).decode()}'
+
+ def _parse_b64(self, encoded_str):
+ try:
+ username, password = b64decode(encoded_str).decode().split(':', 1)
+ except:
+ # fallback in case the token is not bas64 encoded
+ username, password = encoded_str.split(':', 1)
+
+ if not username or not password:
+ raise DecodeError("authentication token does not seems to be b64 encoded")
+
+ return username, password
+
+ def decode(self, encoded_str):
+ """Decode an encrypted HTTP basic authentication string. Returns a tuple of
+ the form (username, password), and raises a DecodeError exception if
+ nothing could be decoded.
+ """
+ split = encoded_str.strip().split(' ')
+
+ # If split is only one element, try to decode the username and password
+ # directly.
+ if len(split) == 1:
+ username, password = self._parse_b64(split[0])
+
+ # If there are only two elements, check the first and ensure it says
+ # 'basic' so that we know we're about to decode the right thing. If not,
+ # bail out.
+ elif len(split) == 2:
+ if split[0].strip().lower() == 'basic':
+ username, password = self._parse_b64(split[1])
+ else:
+ raise DecodeError("authentication token provides more than 2 elements. could not parse properly")
+
+ # If there are more than 2 elements, something crazy must be happening.
+ # Bail.
+ else:
+ raise DecodeError("unpredictable error parsing authentication token")
+
+ return unquote(username), unquote(password)
+
+ def fetch_subject(self, uuid: str, key: str):
+ """
+ Query the internal couchdb searching for the subject mathing the given uuid, key.
+ Normally these stored in wsk or wsku in the form uuid:key
+ :param uuid the OW subject uuid
+ :param key the OW subject key
+ :return a ubject document
+ """
+ print(f"searching for openwhisk subject {uuid}")
+ try:
+ selector= {
+ "selector": {"namespaces": {"$elemMatch": {"uuid": uuid,"key": key}}}
+ }
+
+ response = self._db.find_doc(SUBJECT_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris namespace for user {uuid} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {uuid} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {uuid}. Reason: {e}")
+ return None
+
+ def fetch_user_data(self, username: str):
+ """
+ Query the internal couchdb searching for the given principal to retrieve all the
+ relevant metadata
+ """
+ print(f"searching for user {username} meta-data")
+ try:
+ selector = {"selector":{"login": {"$eq": username }}}
+ response = self._db.find_doc(USER_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris metadata for user {username} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {username} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {username}. Reason: {e}")
+ return None
+
+ def login(self, authorization: str):
+ """
+ Attempt to login the user identified by the given Openwhisk authorization AUTH token as base64
+ """
+ uuid, key = self.decode(authorization)
+ subject = self.fetch_subject(uuid, key)
+
+ if not subject:
+ raise AuthorizationError("Openwhisk subject not found.")
+
+ user_data = self.fetch_user_data(subject['subject'])
+
+ if user_data:
+ return user_data
+
+ raise AuthorizationError("Could not retrieve user metadata.")
\ No newline at end of file
diff --git a/actions/common/command_data.py b/actions/common/command_data.py
new file mode 100644
index 0000000..44f3faa
--- /dev/null
+++ b/actions/common/command_data.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+import logging
+
+class CommandData:
+ _data = {}
+ _raw_data = {}
+
+ def __init__(self, cmd):
+ self._data = cmd
+ self._raw_data = cmd.copy()
+
+ def dump(self):
+ logging.debug(json.dumps(self._data))
+
+ def status(self, status):
+ self._data['status']=status
+
+ def result(self, result):
+ self._data['result']=result
+
+ def get_metadata(self):
+ return self._data
+
+ def get_raw_data(self):
+ return self._raw_data
+
+ def command(self):
+ return self._data['command']
+
+ def args(self):
+ return self._data['args']
+
+
diff --git a/actions/common/minio_util.py b/actions/common/minio_util.py
new file mode 100644
index 0000000..e1e8f90
--- /dev/null
+++ b/actions/common/minio_util.py
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import base64
+import random
+import string
+import mimetypes
+
+from minio import Minio
+from minio.error import S3Error
+from minio.commonconfig import CopySource
+
+def extract_mimetype(file):
+ mimetype, _ = mimetypes.guess_type(file)
+
+ if mimetype is None:
+ return "application/octet-stream"
+ else:
+ return mimetype
+
+def build_mo_client(host, port, access_key, secret_key):
+ """
+ Creates an Minio client pointing to the given MINIO HOST
+ :param host, minio host
+ :param port, minio api port normally it is the 9000
+ :param access_key user we are representing
+ :param secret_key to access minio
+ """
+ mo_client = Minio(f"{host}:{port}",access_key=access_key,secret_key=secret_key,secure=False)
+ return mo_client
+
+def upload_file(mo_client, file, bucket, object_name=None):
+ """Upload a file to an S3 bucket
+
+ :param file_name: File to upload
+ :param bucket: Bucket to upload to
+ :param object_name: S3 object name. If not specified then file_name is used
+ :return: a tuple with the Booleand str,
+ """
+
+ # If S3 object_name was not specified, use file_name
+ if object_name is None:
+ object_name = os.path.basename(file)
+
+ print(f"uploading {object_name} into bucket {bucket} from tmp_file {file}")
+
+ # Upload the file
+ try:
+ mimetype = extract_mimetype(object_name)
+ response = mo_client.fput_object(bucket_name=bucket,object_name=object_name,file_path=file,content_type=mimetype)
+ if response._object_name:
+ return True, response._object_name
+ except Exception as e:
+ print(e)
+ return False, str(e)
+ return False, "KO"
+
+def prepare_file_upload(username, filename, file_content_as_b64):
+ """ Creates a tmp area for the given user where the uploaded file will be stored under a random generated name
+ the fully qualified filename is taken into account only on the corresponding destination bucket.
+ param: username
+ param: filename
+ param: file_content_as_b64
+ return: a file object pointing to the tmp file
+ """
+ try:
+ user_tmp_folder = f"/tmp/{username}"
+ if not os.path.exists(user_tmp_folder):
+ os.makedirs(user_tmp_folder)
+ print(f"added tmp folder {user_tmp_folder}")
+
+ delete_files_in_directory_and_subdirectories(user_tmp_folder)
+ rnd_filename = get_random_string(20)
+ tmp_file = f"{user_tmp_folder}/{rnd_filename}"
+
+ with open(tmp_file, "wb") as f:
+ file_content=base64.b64decode(file_content_as_b64)
+ f.write(file_content)
+
+ if os.path.exists(tmp_file):
+ print(f"{tmp_file} stored successfully.")
+ return tmp_file
+ else:
+ return None
+ except Exception as e:
+ print("error preparing tmp_files",e)
+ return None
+
+def delete_files_in_directory_and_subdirectories(directory_path):
+ try:
+ for root, dirs, files in os.walk(directory_path):
+ for file in files:
+ file_path = os.path.join(root, file)
+ os.remove(file_path)
+ print("All files and subdirectories deleted successfully.")
+ except OSError:
+ print("Error occurred while deleting files and subdirectories.")
+
+def get_random_string(length):
+ # choose from all lowercase letter
+ letters = string.ascii_lowercase
+ return ''.join(random.choice(letters) for i in range(length))
+
+def rm_file(mo_client, bucket, file):
+ """ Remove a file from a bucket
+ :parma mo_client a minio client instance to execute the command
+ :param bucket the bucket name
+ :param file the file name
+ :return True if the file has been removed, False otherwise
+ """
+
+ try:
+ mo_client.remove_object(bucket, file)
+ return True
+ except Exception as e:
+ print(e)
+ return False
+ return False
+
+def cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Copy a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+ try:
+ # copy an object from a bucket to another.
+ cp_result = mo_client.copy_object(
+ dest_bucket,
+ dest_file,
+ CopySource(orig_bucket, orig_file)
+ )
+ if cp_result.object_name:
+ return cp_result.object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
+def mv_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Move a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+
+ # Upload the file
+ try:
+ # mv an object from a bucket to another.
+ object_name = cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file)
+ if object_name:
+ rm_file(mo_client,orig_bucket,orig_file)
+ return object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
diff --git a/actions/common/util.py b/actions/common/util.py
new file mode 100644
index 0000000..f1e8f39
--- /dev/null
+++ b/actions/common/util.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import mimetypes
+
+def get_env_value(user_data, key):
+ """
+ Check if inside the given user_data object there is an env item with the
+ given name and returns the value if any.
+ :param user_data
+ :param key
+ :return None if the given key it is not prese
+ """
+ envs = list(user_data['env'])
+
+ for env in envs:
+ if env['key'] == key:
+ return env['value']
+
+ return None
\ No newline at end of file
diff --git a/actions/content/__main__.py b/actions/content/__main__.py
new file mode 100644
index 0000000..908868a
--- /dev/null
+++ b/actions/content/__main__.py
@@ -0,0 +1,110 @@
+import os
+import base64
+import random
+import string
+import mimetypes
+
+import common.minio_util as mutil
+
+def build_error(message: str):
+ return {
+ "statusCode": 400,
+ "body": message
+ }
+
+def build_response(user, filename, bucket, upload_result, operation):
+ body = {
+ "user":user,
+ "filename":filename,
+ "bucket":bucket,
+ "operation": operation
+ }
+
+ return {
+ "statusCode": upload_result and 200 or 400,
+ "body": body
+ }
+
+
+def process_path_param(ow_path: str):
+ """ Process the __ow_path parameters to extract username and fully qualified filename
+
+ :parma ow_path, the __ow_path parameter passed by openwhisk action controller
+ :return a dictionary {user:<user>, filename:<filepath>}
+ """
+ path_params = ow_path
+ if ow_path.startswith("/"):
+ path_params = ow_path[1:None]
+
+ path_elements = path_params.split("/")
+
+ upload_data = {}
+
+ if len(path_elements) >= 1:
+ upload_data['user']=path_elements[0]
+ upload_data['filename']=path_params.replace(f"{upload_data['user']}/","")
+
+ return upload_data
+
+def _put(mo_client, upload_data, content_as_b64):
+ """
+ Store the given payload into a folder
+ :param mo_client
+ :param upload_data
+ :param content_as_b64 file to be stored in a base64 endoded format
+ """
+ print(f"processing request to upload file {upload_data['filename']} under {upload_data['user']} web-bucket")
+ tmp_file = mutil.prepare_file_upload(upload_data['user'],upload_data['filename'],content_as_b64)
+
+ if tmp_file:
+ upload_result, upload_message = mutil.upload_file(mo_client,tmp_file,f"{upload_data['user']}-web",upload_data['filename'])
+ return build_response(upload_data['user'],upload_data['filename'],f"{upload_data['user']}-web",upload_result,"PUT")
+ else:
+ return build_error("Unexptected error content action. Check activation log")
+
+def _delete(mo_client, upload_data):
+ print(f"processing request to remove file {upload_data['filename']} under {upload_data['user']} web-bucket")
+ removed = mutil.rm_file(mo_client,f"{upload_data['user']}-web", upload_data['filename'])
+
+ if removed:
+ return build_response(upload_data['user'],upload_data['filename'],f"{upload_data['user']}-web",removed,"DELETE")
+ else:
+ return build_error("Unexptected error content action. Check activation log")
+
+def main(args):
+ """
+ Simple actions to upload/remove a files into a minio bucket.
+ The action it is supposed to receive a path param similar to /<user>/<path>?<auth>
+ and will store/remove the given path under the <user>-web bucket using minio client
+ """
+ print(args)
+
+ method = args['__ow_method']
+ headers = args['__ow_headers']
+
+ if(method.lower() not in ['put','delete'] ):
+ return build_error(f"invalid request, HTTP verb {method} is not supported")
+
+ if('minioauth' not in headers):
+ return build_error("invalid request, missing mandatory header: minioauth")
+
+ upload_data = process_path_param(args['__ow_path'])
+
+ if 'user' not in upload_data and 'filename' not in upload_data:
+ return build_error("invalid request, username and/or filename path error")
+
+ minio_host = args['minio_host']
+ minio_port = args['minio_port']
+ auth = headers['minioauth']
+
+ mo_client = mutil.build_mo_client(minio_host, minio_port,upload_data['user'], auth)
+
+ if method.lower() in 'put':
+ if(len(args['__ow_body']) == 0):
+ return build_error("invalid request, no file content has been received")
+
+ content_as_b64 = args['__ow_body']
+ return _put(mo_client, upload_data, content_as_b64)
+
+ if method.lower() in 'delete':
+ return _delete(mo_client, upload_data)
diff --git a/actions/content/common/minio_util.py b/actions/content/common/minio_util.py
new file mode 100644
index 0000000..e1e8f90
--- /dev/null
+++ b/actions/content/common/minio_util.py
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import base64
+import random
+import string
+import mimetypes
+
+from minio import Minio
+from minio.error import S3Error
+from minio.commonconfig import CopySource
+
+def extract_mimetype(file):
+ mimetype, _ = mimetypes.guess_type(file)
+
+ if mimetype is None:
+ return "application/octet-stream"
+ else:
+ return mimetype
+
+def build_mo_client(host, port, access_key, secret_key):
+ """
+ Creates an Minio client pointing to the given MINIO HOST
+ :param host, minio host
+ :param port, minio api port normally it is the 9000
+ :param access_key user we are representing
+ :param secret_key to access minio
+ """
+ mo_client = Minio(f"{host}:{port}",access_key=access_key,secret_key=secret_key,secure=False)
+ return mo_client
+
+def upload_file(mo_client, file, bucket, object_name=None):
+ """Upload a file to an S3 bucket
+
+ :param file_name: File to upload
+ :param bucket: Bucket to upload to
+ :param object_name: S3 object name. If not specified then file_name is used
+ :return: a tuple with the Booleand str,
+ """
+
+ # If S3 object_name was not specified, use file_name
+ if object_name is None:
+ object_name = os.path.basename(file)
+
+ print(f"uploading {object_name} into bucket {bucket} from tmp_file {file}")
+
+ # Upload the file
+ try:
+ mimetype = extract_mimetype(object_name)
+ response = mo_client.fput_object(bucket_name=bucket,object_name=object_name,file_path=file,content_type=mimetype)
+ if response._object_name:
+ return True, response._object_name
+ except Exception as e:
+ print(e)
+ return False, str(e)
+ return False, "KO"
+
+def prepare_file_upload(username, filename, file_content_as_b64):
+ """ Creates a tmp area for the given user where the uploaded file will be stored under a random generated name
+ the fully qualified filename is taken into account only on the corresponding destination bucket.
+ param: username
+ param: filename
+ param: file_content_as_b64
+ return: a file object pointing to the tmp file
+ """
+ try:
+ user_tmp_folder = f"/tmp/{username}"
+ if not os.path.exists(user_tmp_folder):
+ os.makedirs(user_tmp_folder)
+ print(f"added tmp folder {user_tmp_folder}")
+
+ delete_files_in_directory_and_subdirectories(user_tmp_folder)
+ rnd_filename = get_random_string(20)
+ tmp_file = f"{user_tmp_folder}/{rnd_filename}"
+
+ with open(tmp_file, "wb") as f:
+ file_content=base64.b64decode(file_content_as_b64)
+ f.write(file_content)
+
+ if os.path.exists(tmp_file):
+ print(f"{tmp_file} stored successfully.")
+ return tmp_file
+ else:
+ return None
+ except Exception as e:
+ print("error preparing tmp_files",e)
+ return None
+
+def delete_files_in_directory_and_subdirectories(directory_path):
+ try:
+ for root, dirs, files in os.walk(directory_path):
+ for file in files:
+ file_path = os.path.join(root, file)
+ os.remove(file_path)
+ print("All files and subdirectories deleted successfully.")
+ except OSError:
+ print("Error occurred while deleting files and subdirectories.")
+
+def get_random_string(length):
+ # choose from all lowercase letter
+ letters = string.ascii_lowercase
+ return ''.join(random.choice(letters) for i in range(length))
+
+def rm_file(mo_client, bucket, file):
+ """ Remove a file from a bucket
+ :parma mo_client a minio client instance to execute the command
+ :param bucket the bucket name
+ :param file the file name
+ :return True if the file has been removed, False otherwise
+ """
+
+ try:
+ mo_client.remove_object(bucket, file)
+ return True
+ except Exception as e:
+ print(e)
+ return False
+ return False
+
+def cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Copy a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+ try:
+ # copy an object from a bucket to another.
+ cp_result = mo_client.copy_object(
+ dest_bucket,
+ dest_file,
+ CopySource(orig_bucket, orig_file)
+ )
+ if cp_result.object_name:
+ return cp_result.object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
+def mv_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Move a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+
+ # Upload the file
+ try:
+ # mv an object from a bucket to another.
+ object_name = cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file)
+ if object_name:
+ rm_file(mo_client,orig_bucket,orig_file)
+ return object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
diff --git a/actions/devel/download/__main__.py b/actions/devel/download/__main__.py
new file mode 100644
index 0000000..080f45b
--- /dev/null
+++ b/actions/devel/download/__main__.py
@@ -0,0 +1,91 @@
+import os
+import base64
+import random
+import string
+import mimetypes
+import io
+
+import common.minio_util as mutil
+import common.util as ut
+
+from common.authorize import Authorize
+
+def build_error(message: str):
+ return {
+ "statusCode": 400,
+ "body": message
+ }
+
+def build_response(filename, content_type, buffer):
+ return {
+ "statusCode": 200,
+ "headers": {"Content-Type":content_type, "Content-Disposition": f"inline; filename='{filename}'"},
+ "body": buffer.getvalue().decode()
+ }
+
+
+def process_path_param(ow_path: str):
+ """ Process the __ow_path parameters to extract username and fully qualified filename
+
+ :parma ow_path, the __ow_path parameter passed by openwhisk action controller
+ :return a dictionary {bucket:<bucket>, filename:<filepath>}
+ """
+ path_params = ow_path
+ if ow_path.startswith("/"):
+ path_params = ow_path[1:None]
+
+ path_elements = path_params.split("/")
+
+ upload_data = {}
+
+ if len(path_elements) >= 1:
+ upload_data['bucket']=path_elements[0]
+ upload_data['path']=path_params.replace(f"{upload_data['bucket']}/","")
+
+ file_elements = upload_data['path'].split("/")
+
+ if len(file_elements) > 1:
+ upload_data['filename']=file_elements[len(file_elements)-1]
+ else:
+ upload_data['filename']=file_elements
+
+ return upload_data
+
+
+def main(args):
+ """
+ Action implementing a generic download wrapper for the nuv devel plugin. The invoker must provide a x-impersonate-auth header containing the Openwhisk BASIC authentication of the wsku/user the action should impersonate
+ when calling this action. The upload action it is supposed to receive a path param similar to /<bucket>/<path>
+ and will attempt to retrieve the given path under the given MINIO <bucket>. The bucket must exists and the impersonated user must have read permission on it.
+ """
+ headers = args['__ow_headers']
+ if('x-impersonate-auth' not in headers):
+ return build_error("invalid request, missing mandatory header: x-impersonate-auth")
+
+ try:
+ download_data = process_path_param(args['__ow_path'])
+
+ if 'bucket' not in download_data and 'filename' not in download_data:
+ return build_error("invalid request, bucket and/or filename path error")
+
+ print(f"processing request to download {download_data['path']} from bucket {download_data['bucket']}")
+ user_data = Authorize(args['couchdb_host'],args['couchdb_user'],args['couchdb_password']).login(headers['x-impersonate-auth'])
+
+ mo_client = mutil.build_mo_client(ut.get_env_value(user_data,"MINIO_HOST"), ut.get_env_value(user_data,"MINIO_PORT"),ut.get_env_value(user_data,"MINIO_ACCESS_KEY") , ut.get_env_value(user_data,"MINIO_SECRET_KEY"))
+
+ # see https://urllib3.readthedocs.io/en/latest/reference/urllib3.response.html for the format
+ response = mo_client.get_object(bucket_name = download_data['bucket'], object_name= download_data['path'])
+
+ content_type = response.getheader('Content-Type')
+ buffer = io.BytesIO()
+
+ for d in response.stream():
+ buffer.write(d)
+
+ return build_response(download_data['filename'], content_type, buffer)
+ except Exception as e:
+ return build_error(f"failed to execute nuv devel command. Reason: {e}")
+ finally:
+ if response:
+ response.close()
+ response.release_conn()
diff --git a/actions/devel/download/common/authorize.py b/actions/devel/download/common/authorize.py
new file mode 100644
index 0000000..f7d716f
--- /dev/null
+++ b/actions/devel/download/common/authorize.py
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from base64 import b64decode, b64encode
+from urllib.parse import quote, unquote
+
+import nuvolaris.config as cfg
+import nuvolaris.couchdb_util as cu
+import json
+import common.util as ut
+
+USER_META_DBN = "users_metadata"
+SUBJECT_META_DBN = "subjects"
+
+class DecodeError(Exception):
+ pass
+
+class EncodeError(Exception):
+ pass
+
+class AuthorizationError(Exception):
+ pass
+
+class Authorize():
+
+ def __init__(self, cdb_host, cdb_user, cdb_pwd):
+ cfg.clean()
+ cfg.put("couchdb.host", cdb_host)
+ cfg.put("couchdb.admin.user", cdb_user)
+ cfg.put("couchdb.admin.password", cdb_pwd)
+ self._db = cu.CouchDB()
+
+ def encode(self,username, password):
+ """Returns an HTTP basic authentication encrypted string given a valid
+ username and password.
+ """
+ if ':' in username:
+ raise EncodeError
+
+ username_password = f'{quote(username)}:{quote(password)}'
+ return f'Basic {b64encode(username_password.encode()).decode()}'
+
+ def _parse_b64(self, encoded_str):
+ try:
+ username, password = b64decode(encoded_str).decode().split(':', 1)
+ except:
+ # fallback in case the token is not bas64 encoded
+ username, password = encoded_str.split(':', 1)
+
+ if not username or not password:
+ raise DecodeError("authentication token does not seems to be b64 encoded")
+
+ return username, password
+
+ def decode(self, encoded_str):
+ """Decode an encrypted HTTP basic authentication string. Returns a tuple of
+ the form (username, password), and raises a DecodeError exception if
+ nothing could be decoded.
+ """
+ split = encoded_str.strip().split(' ')
+
+ # If split is only one element, try to decode the username and password
+ # directly.
+ if len(split) == 1:
+ username, password = self._parse_b64(split[0])
+
+ # If there are only two elements, check the first and ensure it says
+ # 'basic' so that we know we're about to decode the right thing. If not,
+ # bail out.
+ elif len(split) == 2:
+ if split[0].strip().lower() == 'basic':
+ username, password = self._parse_b64(split[1])
+ else:
+ raise DecodeError("authentication token provides more than 2 elements. could not parse properly")
+
+ # If there are more than 2 elements, something crazy must be happening.
+ # Bail.
+ else:
+ raise DecodeError("unpredictable error parsing authentication token")
+
+ return unquote(username), unquote(password)
+
+ def fetch_subject(self, uuid: str, key: str):
+ """
+ Query the internal couchdb searching for the subject mathing the given uuid, key.
+ Normally these stored in wsk or wsku in the form uuid:key
+ :param uuid the OW subject uuid
+ :param key the OW subject key
+ :return a ubject document
+ """
+ print(f"searching for openwhisk subject {uuid}")
+ try:
+ selector= {
+ "selector": {"namespaces": {"$elemMatch": {"uuid": uuid,"key": key}}}
+ }
+
+ response = self._db.find_doc(SUBJECT_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris namespace for user {uuid} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {uuid} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {uuid}. Reason: {e}")
+ return None
+
+ def fetch_user_data(self, username: str):
+ """
+ Query the internal couchdb searching for the given principal to retrieve all the
+ relevant metadata
+ """
+ print(f"searching for user {username} meta-data")
+ try:
+ selector = {"selector":{"login": {"$eq": username }}}
+ response = self._db.find_doc(USER_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris metadata for user {username} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {username} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {username}. Reason: {e}")
+ return None
+
+ def login(self, authorization: str):
+ """
+ Attempt to login the user identified by the given Openwhisk authorization AUTH token as base64
+ """
+ uuid, key = self.decode(authorization)
+ subject = self.fetch_subject(uuid, key)
+
+ if not subject:
+ raise AuthorizationError("Openwhisk subject not found.")
+
+ user_data = self.fetch_user_data(subject['subject'])
+
+ if user_data:
+ return user_data
+
+ raise AuthorizationError("Could not retrieve user metadata.")
\ No newline at end of file
diff --git a/actions/devel/download/common/command_data.py b/actions/devel/download/common/command_data.py
new file mode 100644
index 0000000..44f3faa
--- /dev/null
+++ b/actions/devel/download/common/command_data.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+import logging
+
+class CommandData:
+ _data = {}
+ _raw_data = {}
+
+ def __init__(self, cmd):
+ self._data = cmd
+ self._raw_data = cmd.copy()
+
+ def dump(self):
+ logging.debug(json.dumps(self._data))
+
+ def status(self, status):
+ self._data['status']=status
+
+ def result(self, result):
+ self._data['result']=result
+
+ def get_metadata(self):
+ return self._data
+
+ def get_raw_data(self):
+ return self._raw_data
+
+ def command(self):
+ return self._data['command']
+
+ def args(self):
+ return self._data['args']
+
+
diff --git a/actions/devel/download/common/minio_util.py b/actions/devel/download/common/minio_util.py
new file mode 100644
index 0000000..e1e8f90
--- /dev/null
+++ b/actions/devel/download/common/minio_util.py
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import base64
+import random
+import string
+import mimetypes
+
+from minio import Minio
+from minio.error import S3Error
+from minio.commonconfig import CopySource
+
+def extract_mimetype(file):
+ mimetype, _ = mimetypes.guess_type(file)
+
+ if mimetype is None:
+ return "application/octet-stream"
+ else:
+ return mimetype
+
+def build_mo_client(host, port, access_key, secret_key):
+ """
+ Creates an Minio client pointing to the given MINIO HOST
+ :param host, minio host
+ :param port, minio api port normally it is the 9000
+ :param access_key user we are representing
+ :param secret_key to access minio
+ """
+ mo_client = Minio(f"{host}:{port}",access_key=access_key,secret_key=secret_key,secure=False)
+ return mo_client
+
+def upload_file(mo_client, file, bucket, object_name=None):
+ """Upload a file to an S3 bucket
+
+ :param file_name: File to upload
+ :param bucket: Bucket to upload to
+ :param object_name: S3 object name. If not specified then file_name is used
+ :return: a tuple with the Booleand str,
+ """
+
+ # If S3 object_name was not specified, use file_name
+ if object_name is None:
+ object_name = os.path.basename(file)
+
+ print(f"uploading {object_name} into bucket {bucket} from tmp_file {file}")
+
+ # Upload the file
+ try:
+ mimetype = extract_mimetype(object_name)
+ response = mo_client.fput_object(bucket_name=bucket,object_name=object_name,file_path=file,content_type=mimetype)
+ if response._object_name:
+ return True, response._object_name
+ except Exception as e:
+ print(e)
+ return False, str(e)
+ return False, "KO"
+
+def prepare_file_upload(username, filename, file_content_as_b64):
+ """ Creates a tmp area for the given user where the uploaded file will be stored under a random generated name
+ the fully qualified filename is taken into account only on the corresponding destination bucket.
+ param: username
+ param: filename
+ param: file_content_as_b64
+ return: a file object pointing to the tmp file
+ """
+ try:
+ user_tmp_folder = f"/tmp/{username}"
+ if not os.path.exists(user_tmp_folder):
+ os.makedirs(user_tmp_folder)
+ print(f"added tmp folder {user_tmp_folder}")
+
+ delete_files_in_directory_and_subdirectories(user_tmp_folder)
+ rnd_filename = get_random_string(20)
+ tmp_file = f"{user_tmp_folder}/{rnd_filename}"
+
+ with open(tmp_file, "wb") as f:
+ file_content=base64.b64decode(file_content_as_b64)
+ f.write(file_content)
+
+ if os.path.exists(tmp_file):
+ print(f"{tmp_file} stored successfully.")
+ return tmp_file
+ else:
+ return None
+ except Exception as e:
+ print("error preparing tmp_files",e)
+ return None
+
+def delete_files_in_directory_and_subdirectories(directory_path):
+ try:
+ for root, dirs, files in os.walk(directory_path):
+ for file in files:
+ file_path = os.path.join(root, file)
+ os.remove(file_path)
+ print("All files and subdirectories deleted successfully.")
+ except OSError:
+ print("Error occurred while deleting files and subdirectories.")
+
+def get_random_string(length):
+ # choose from all lowercase letter
+ letters = string.ascii_lowercase
+ return ''.join(random.choice(letters) for i in range(length))
+
+def rm_file(mo_client, bucket, file):
+ """ Remove a file from a bucket
+ :parma mo_client a minio client instance to execute the command
+ :param bucket the bucket name
+ :param file the file name
+ :return True if the file has been removed, False otherwise
+ """
+
+ try:
+ mo_client.remove_object(bucket, file)
+ return True
+ except Exception as e:
+ print(e)
+ return False
+ return False
+
+def cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Copy a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+ try:
+ # copy an object from a bucket to another.
+ cp_result = mo_client.copy_object(
+ dest_bucket,
+ dest_file,
+ CopySource(orig_bucket, orig_file)
+ )
+ if cp_result.object_name:
+ return cp_result.object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
+def mv_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Move a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+
+ # Upload the file
+ try:
+ # mv an object from a bucket to another.
+ object_name = cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file)
+ if object_name:
+ rm_file(mo_client,orig_bucket,orig_file)
+ return object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
diff --git a/actions/devel/download/common/util.py b/actions/devel/download/common/util.py
new file mode 100644
index 0000000..f1e8f39
--- /dev/null
+++ b/actions/devel/download/common/util.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import mimetypes
+
+def get_env_value(user_data, key):
+ """
+ Check if inside the given user_data object there is an env item with the
+ given name and returns the value if any.
+ :param user_data
+ :param key
+ :return None if the given key it is not prese
+ """
+ envs = list(user_data['env'])
+
+ for env in envs:
+ if env['key'] == key:
+ return env['value']
+
+ return None
\ No newline at end of file
diff --git a/actions/devel/download/flatdict.py b/actions/devel/download/flatdict.py
new file mode 100644
index 0000000..834ce99
--- /dev/null
+++ b/actions/devel/download/flatdict.py
@@ -0,0 +1,506 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""FlatDict is a dict object that allows for single level, delimited
+key/value pair mapping of nested dictionaries.
+
+"""
+try:
+ from collections.abc import MutableMapping
+except ImportError: # pragma: nocover
+ from collections import MutableMapping
+import sys
+
+__version__ = '4.0.1'
+
+NO_DEFAULT = object()
+
+
+class FlatDict(MutableMapping):
+ """:class:`~flatdict.FlatDict` is a dictionary object that allows for
+ single level, delimited key/value pair mapping of nested dictionaries.
+ The default delimiter value is ``:`` but can be changed in the constructor
+ or by calling :meth:`FlatDict.set_delimiter`.
+
+ """
+ _COERCE = dict
+
+ def __init__(self, value=None, delimiter=':', dict_class=dict):
+ super(FlatDict, self).__init__()
+ self._values = dict_class()
+ self._delimiter = delimiter
+ self.update(value)
+
+ def __contains__(self, key):
+ """Check to see if the key exists, checking for both delimited and
+ not delimited key values.
+
+ :param mixed key: The key to check for
+
+ """
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ return pk in self._values and ck in self._values[pk]
+ return key in self._values
+
+ def __delitem__(self, key):
+ """Delete the item for the specified key, automatically dealing with
+ nested children.
+
+ :param mixed key: The key to use
+ :raises: KeyError
+
+ """
+ if key not in self:
+ raise KeyError
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ del self._values[pk][ck]
+ if not self._values[pk]:
+ del self._values[pk]
+ else:
+ del self._values[key]
+
+ def __eq__(self, other):
+ """Check for equality against the other value
+
+ :param other: The value to compare
+ :type other: FlatDict
+ :rtype: bool
+ :raises: TypeError
+
+ """
+ if isinstance(other, dict):
+ return self.as_dict() == other
+ elif not isinstance(other, self.__class__):
+ raise TypeError
+ return self.as_dict() == other.as_dict()
+
+ def __ne__(self, other):
+ """Check for inequality against the other value
+
+ :param other: The value to compare
+ :type other: dict or FlatDict
+ :rtype: bool
+
+ """
+ return not self.__eq__(other)
+
+ def __getitem__(self, key):
+ """Get an item for the specified key, automatically dealing with
+ nested children.
+
+ :param mixed key: The key to use
+ :rtype: mixed
+ :raises: KeyError
+
+ """
+ values = self._values
+ key = [key] if isinstance(key, int) else key.split(self._delimiter)
+ for part in key:
+ values = values[part]
+ return values
+
+ def __iter__(self):
+ """Iterate over the flat dictionary key and values
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ return iter(self.keys())
+
+ def __len__(self):
+ """Return the number of items.
+
+ :rtype: int
+
+ """
+ return len(self.keys())
+
+ def __reduce__(self):
+ """Return state information for pickling
+
+ :rtype: tuple
+
+ """
+ return type(self), (self.as_dict(), self._delimiter)
+
+ def __repr__(self):
+ """Return the string representation of the instance.
+
+ :rtype: str
+
+ """
+ return '<{} id={} {}>"'.format(self.__class__.__name__, id(self),
+ str(self))
+
+ def __setitem__(self, key, value):
+ """Assign the value to the key, dynamically building nested
+ FlatDict items where appropriate.
+
+ :param mixed key: The key for the item
+ :param mixed value: The value for the item
+ :raises: TypeError
+
+ """
+ if isinstance(value, self._COERCE) and not isinstance(value, FlatDict):
+ value = self.__class__(value, self._delimiter)
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if pk not in self._values:
+ self._values[pk] = self.__class__({ck: value}, self._delimiter)
+ return
+ elif not isinstance(self._values[pk], FlatDict):
+ raise TypeError(
+ 'Assignment to invalid type for key {}'.format(pk))
+ self._values[pk][ck] = value
+ else:
+ self._values[key] = value
+
+ def __str__(self):
+ """Return the string value of the instance.
+
+ :rtype: str
+
+ """
+ return '{{{}}}'.format(', '.join(
+ ['{!r}: {!r}'.format(k, self[k]) for k in self.keys()]))
+
+ def as_dict(self):
+ """Return the :class:`~flatdict.FlatDict` as a :class:`dict`
+
+ :rtype: dict
+
+ """
+ out = dict({})
+ for key in self.keys():
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if self._has_delimiter(ck):
+ ck = ck.split(self._delimiter, 1)[0]
+ if isinstance(self._values[pk], FlatDict) and pk not in out:
+ out[pk] = {}
+ if isinstance(self._values[pk][ck], FlatDict):
+ out[pk][ck] = self._values[pk][ck].as_dict()
+ else:
+ out[pk][ck] = self._values[pk][ck]
+ else:
+ out[key] = self._values[key]
+ return out
+
+ def clear(self):
+ """Remove all items from the flat dictionary."""
+ self._values.clear()
+
+ def copy(self):
+ """Return a shallow copy of the flat dictionary.
+
+ :rtype: flatdict.FlatDict
+
+ """
+ return self.__class__(self.as_dict(), delimiter=self._delimiter)
+
+ def get(self, key, d=None):
+ """Return the value for key if key is in the flat dictionary, else
+ default. If default is not given, it defaults to ``None``, so that this
+ method never raises :exc:`KeyError`.
+
+ :param mixed key: The key to get
+ :param mixed d: The default value
+ :rtype: mixed
+
+ """
+ try:
+ return self.__getitem__(key)
+ except KeyError:
+ return d
+
+ def items(self):
+ """Return a copy of the flat dictionary's list of ``(key, value)``
+ pairs.
+
+ .. note:: CPython implementation detail: Keys and values are listed in
+ an arbitrary order which is non-random, varies across Python
+ implementations, and depends on the flat dictionary's history of
+ insertions and deletions.
+
+ :rtype: list
+
+ """
+ return [(k, self.__getitem__(k)) for k in self.keys()]
+
+ def iteritems(self):
+ """Return an iterator over the flat dictionary's (key, value) pairs.
+ See the note for :meth:`flatdict.FlatDict.items`.
+
+ Using ``iteritems()`` while adding or deleting entries in the flat
+ dictionary may raise :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for item in self.items():
+ yield item
+
+ def iterkeys(self):
+ """Iterate over the flat dictionary's keys. See the note for
+ :meth:`flatdict.FlatDict.items`.
+
+ Using ``iterkeys()`` while adding or deleting entries in the flat
+ dictionary may raise :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for key in self.keys():
+ yield key
+
+ def itervalues(self):
+ """Return an iterator over the flat dictionary's values. See the note
+ :meth:`flatdict.FlatDict.items`.
+
+ Using ``itervalues()`` while adding or deleting entries in the flat
+ dictionary may raise a :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for value in self.values():
+ yield value
+
+ def keys(self):
+ """Return a copy of the flat dictionary's list of keys.
+ See the note for :meth:`flatdict.FlatDict.items`.
+
+ :rtype: list
+
+ """
+ keys = []
+
+ for key, value in self._values.items():
+ if isinstance(value, (FlatDict, dict)):
+ nested = [
+ self._delimiter.join([str(key), str(k)])
+ for k in value.keys()]
+ keys += nested if nested else [key]
+ else:
+ keys.append(key)
+
+ return keys
+
+ def pop(self, key, default=NO_DEFAULT):
+ """If key is in the flat dictionary, remove it and return its value,
+ else return default. If default is not given and key is not in the
+ dictionary, :exc:`KeyError` is raised.
+
+ :param mixed key: The key name
+ :param mixed default: The default value
+ :rtype: mixed
+
+ """
+ if key not in self and default != NO_DEFAULT:
+ return default
+ value = self[key]
+ self.__delitem__(key)
+ return value
+
+ def setdefault(self, key, default):
+ """If key is in the flat dictionary, return its value. If not,
+ insert key with a value of default and return default.
+ default defaults to ``None``.
+
+ :param mixed key: The key name
+ :param mixed default: The default value
+ :rtype: mixed
+
+ """
+ if key not in self:
+ self.__setitem__(key, default)
+ return self.__getitem__(key)
+
+ def set_delimiter(self, delimiter):
+ """Override the default or passed in delimiter with a new value. If
+ the requested delimiter already exists in a key, a :exc:`ValueError`
+ will be raised.
+
+ :param str delimiter: The delimiter to use
+ :raises: ValueError
+
+ """
+ for key in self.keys():
+ if delimiter in key:
+ raise ValueError('Key {!r} collides with delimiter {!r}', key,
+ delimiter)
+ self._delimiter = delimiter
+ for key in self._values.keys():
+ if isinstance(self._values[key], FlatDict):
+ self._values[key].set_delimiter(delimiter)
+
+ def update(self, other=None, **kwargs):
+ """Update the flat dictionary with the key/value pairs from other,
+ overwriting existing keys.
+
+ ``update()`` accepts either another flat dictionary object or an
+ iterable of key/value pairs (as tuples or other iterables of length
+ two). If keyword arguments are specified, the flat dictionary is then
+ updated with those key/value pairs: ``d.update(red=1, blue=2)``.
+
+ :param iterable other: Iterable of key, value pairs
+ :rtype: None
+
+ """
+ [self.__setitem__(k, v) for k, v in dict(other or kwargs).items()]
+
+ def values(self):
+ """Return a copy of the flat dictionary's list of values. See the note
+ for :meth:`flatdict.FlatDict.items`.
+
+ :rtype: list
+
+ """
+ return [self.__getitem__(k) for k in self.keys()]
+
+ def _has_delimiter(self, key):
+ """Checks to see if the key contains the delimiter.
+
+ :rtype: bool
+
+ """
+ return isinstance(key, str) and self._delimiter in key
+
+
+class FlatterDict(FlatDict):
+ """Like :class:`~flatdict.FlatDict` but also coerces lists and sets
+ to child-dict instances with the offset as the key. Alternative to
+ the implementation added in v1.2 of FlatDict.
+
+ """
+ _COERCE = list, tuple, set, dict, FlatDict
+ _ARRAYS = list, set, tuple
+
+ def __init__(self, value=None, delimiter=':', dict_class=dict):
+ self.original_type = type(value)
+ if self.original_type in self._ARRAYS:
+ value = {str(i): v for i, v in enumerate(value)}
+ super(FlatterDict, self).__init__(value, delimiter, dict_class)
+
+ def __setitem__(self, key, value):
+ """Assign the value to the key, dynamically building nested
+ FlatDict items where appropriate.
+
+ :param mixed key: The key for the item
+ :param mixed value: The value for the item
+ :raises: TypeError
+
+ """
+ if isinstance(value, self._COERCE) and \
+ not isinstance(value, FlatterDict):
+ value = self.__class__(value, self._delimiter)
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if pk not in self._values:
+ self._values[pk] = self.__class__({ck: value}, self._delimiter)
+ return
+ if getattr(self._values[pk], 'original_type',
+ None) in self._ARRAYS:
+ try:
+ k, cck = ck.split(self._delimiter, 1)
+ int(k)
+ except ValueError:
+ raise TypeError(
+ 'Assignment to invalid type for key {}{}{}'.format(
+ pk, self._delimiter, ck))
+ self._values[pk][k][cck] = value
+ return
+ elif not isinstance(self._values[pk], FlatterDict):
+ raise TypeError(
+ 'Assignment to invalid type for key {}'.format(pk))
+ self._values[pk][ck] = value
+ else:
+ self._values[key] = value
+
+ def as_dict(self):
+ """Return the :class:`~flatdict.FlatterDict` as a nested
+ :class:`dict`.
+
+ :rtype: dict
+
+ """
+ out = {}
+ for key in self.keys():
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if self._has_delimiter(ck):
+ ck = ck.split(self._delimiter, 1)[0]
+ if isinstance(self._values[pk], FlatterDict) and pk not in out:
+ if self._values[pk].original_type == tuple:
+ out[pk] = tuple(self._child_as_list(pk))
+ elif self._values[pk].original_type == list:
+ out[pk] = self._child_as_list(pk)
+ elif self._values[pk].original_type == set:
+ out[pk] = set(self._child_as_list(pk))
+ elif self._values[pk].original_type == dict:
+ out[pk] = self._values[pk].as_dict()
+ else:
+ if isinstance(self._values[key], FlatterDict):
+ out[key] = self._values[key].original_type()
+ else:
+ out[key] = self._values[key]
+ return out
+
+ def _child_as_list(self, pk, ck=None):
+ """Returns a list of values from the child FlatterDict instance
+ with string based integer keys.
+
+ :param str pk: The parent key
+ :param str ck: The child key, optional
+ :rtype: list
+
+ """
+ if ck is None:
+ subset = self._values[pk]
+ else:
+ subset = self._values[pk][ck]
+ # Check if keys has delimiter, which implies deeply nested dict
+ keys = subset.keys()
+ if any(self._has_delimiter(k) for k in keys):
+ out = []
+ split_keys = {k.split(self._delimiter)[0] for k in keys}
+ for k in sorted(split_keys, key=lambda x: int(x)):
+ if subset[k].original_type == tuple:
+ out.append(tuple(self._child_as_list(pk, k)))
+ elif subset[k].original_type == list:
+ out.append(self._child_as_list(pk, k))
+ elif subset[k].original_type == set:
+ out.append(set(self._child_as_list(pk, k)))
+ elif subset[k].original_type == dict:
+ out.append(subset[k].as_dict())
+ return out
+
+ # Python prior 3.6 does not guarantee insertion order, remove it after
+ # EOL python 3.5 - 2020-09-13
+ if sys.version_info[0:2] < (3, 6): # pragma: nocover
+ return [subset[k] for k in sorted(keys, key=lambda x: int(x))]
+ else:
+ return [subset[k] for k in keys]
diff --git a/actions/devel/download/nuvolaris/config.py b/actions/devel/download/nuvolaris/config.py
new file mode 100644
index 0000000..787d4a7
--- /dev/null
+++ b/actions/devel/download/nuvolaris/config.py
@@ -0,0 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import flatdict, json, os
+import logging
+
+_config = {}
+
+# define a configuration
+# the configuration is a map, followed by a list of labels
+# the map can be a serialized json and will be flattened to a map of values.
+# you can have only a configuration active at a time
+# if you want to set a new configuration you have to clean it
+def configure(spec: dict):
+ global _config
+ _config = dict(flatdict.FlatDict(spec, delimiter="."))
+
+ #forces autodetect of kube if not provided
+ if not exists('nuvolaris.kube'):
+ put('nuvolaris.kube','auto')
+
+ return True
+
+def clean():
+ global _config
+ _config = {}
+
+def exists(key):
+ return key in _config
+
+def get(key, envvar=None, defval=None):
+ val = _config.get(key)
+
+ if envvar and envvar in os.environ:
+ val = os.environ[envvar]
+
+ if val:
+ return val
+
+ return defval
+
+def put(key, value):
+ _config[key] = value
+ return True
+
+def delete(key):
+ if key in _config:
+ del _config[key]
+ return True
+
+def getall(prefix=""):
+ res = {}
+ for key in _config.keys():
+ if key.startswith(prefix):
+ res[key] = _config[key]
+ return res
+
+def keys(prefix=""):
+ res = []
+ if _config:
+ for key in _config.keys():
+ if key.startswith(prefix):
+ res.append(key)
+ return res
+
+def detect_labels(labels=None):
+ # skip autodetection of nuvolaris.kube if already available in _config
+ if exists('nuvolaris.kube') and get('nuvolaris.kube') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.kube={get('nuvolaris.kube')}")
+ return {}
+
+ # read labels if not available
+ if not labels:
+ import nuvolaris.kube as kube
+ labels = kube.kubectl("get", "nodes", jsonpath='{.items[].metadata.labels}')
+
+ res = {}
+ kube = None
+ for i in labels:
+ for j in list(i.keys()):
+ if j.startswith("nuvolaris.io/"):
+ key = f"nuvolaris.{j[13:]}"
+ res[key] = i[j]
+ _config[key] = i[j]
+
+ for i in labels:
+ for j in list(i.keys()):
+ # detect the kube type
+ if j.find("eksctl.io") >= 0:
+ kube ="eks"
+ break
+ if j.find("k8s.io/cloud-provider-aws") >= 0:
+ kube ="eks"
+ break
+ elif j.find("microk8s.io") >= 0:
+ kube = "microk8s"
+ break
+ elif j.find("lke.linode.com") >=0:
+ kube = "lks"
+ break
+ elif j.find("node.openshift.io") >=0:
+ kube = "openshift"
+ break
+ elif j.endswith("kubernetes.io/instance-type") and i[j] == "k3s":
+ kube = "k3s"
+ break
+ elif j.find("cloud.google.com/gke") >=0:
+ kube = "gke"
+ break
+ elif j.find("kubernetes.azure.com") >=0:
+ kube = "aks"
+ break
+ # assign all the 'nuvolaris.io' labels
+
+ if kube:
+ res["nuvolaris.kube"] = kube
+ _config["nuvolaris.kube"] = kube
+ return res
+
+ # defaults to generic if it is not yet detected
+ if exists('nuvolaris.kube') and get('nuvolaris.kube') == 'auto':
+ _config["nuvolaris.kube"] = "generic"
+ res["nuvolaris.kube"] = "generic"
+
+ return res
+
+def detect_storage(storages=None):
+ res = {}
+ if not storages:
+ import nuvolaris.util as util
+
+ try:
+ detect_storageclass = True
+ detect_provisioner = True
+
+ # skips autodetection if already provided
+ if exists('nuvolaris.storageclass') and get('nuvolaris.storageclass') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.storageclass={get('nuvolaris.storageclass')}")
+ detect_storageclass = False
+
+ if exists('nuvolaris.provisioner') and get('nuvolaris.provisioner') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.provisioner={get('nuvolaris.provisioner')}")
+ detect_provisioner = False
+
+ if not detect_storageclass and not detect_provisioner:
+ return res
+
+ if detect_storageclass:
+ storage_class = util.get_default_storage_class()
+ if storage_class:
+ res['nuvolaris.storageclass'] = storage_class
+ _config['nuvolaris.storageclass'] = storage_class
+
+ if detect_provisioner:
+ provisioner = util.get_default_storage_provisioner()
+ if(provisioner):
+ res['nuvolaris.provisioner'] = provisioner
+ _config['nuvolaris.provisioner'] = provisioner
+ except:
+ pass
+ return res
+
+def detect_env():
+ _config['operator.image'] = os.environ.get("OPERATOR_IMAGE", "missing-OPERATOR_IMAGE")
+ _config['operator.tag'] = os.environ.get("OPERATOR_TAG", "missing-OPERATOR_TAG")
+
+ # skip autodetection of controller.image if already configure into whisk.yaml
+ if not exists('controller.image'):
+ _config['controller.image'] = os.environ.get("CONTROLLER_IMAGE", "missing-CONTROLLER_IMAGE")
+ _config['controller.tag'] = os.environ.get("CONTROLLER_TAG", "missing-CONTROLLER_TAG")
+ else:
+ logging.warn(f"OW controller image detection skipped. Using {get('controller.image')}")
+
+ if not exists('invoker.image'):
+ _config['invoker.image'] = os.environ.get("INVOKER_IMAGE", "missing-INVOKER_IMAGE")
+ _config['invoker.tag'] = os.environ.get("INVOKER_TAG", "missing-INVOKER_TAG")
+ else:
+ logging.warn(f"OW invoker image detection skipped. Using {get('invoker.image')}")
+
+def detect():
+ detect_storage()
+ detect_labels()
+ detect_env()
+
+def dump_config():
+ import nuvolaris.config as cfg
+ for k in cfg.getall():
+ logging.debug(f"{k} = {cfg.get(k)}")
diff --git a/actions/devel/download/nuvolaris/couchdb_util.py b/actions/devel/download/nuvolaris/couchdb_util.py
new file mode 100644
index 0000000..3adb6b9
--- /dev/null
+++ b/actions/devel/download/nuvolaris/couchdb_util.py
@@ -0,0 +1,164 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import os, json, time, sys, logging
+import requests as req
+import nuvolaris.config as cfg
+
+class CouchDB:
+ def __init__(self):
+ self.db_protocol = "http"
+ self.db_prefix = "nuvolaris_"
+ self.db_port = cfg.get("couchdb.port", "COUCHDB_SERVICE_PORT", "5984")
+ self.db_host = cfg.get("couchdb.host", "COUCHDB_SERVICE_HOST", "couchdb")
+ self.db_username = cfg.get("couchdb.admin.user", "COUCHDB_ADMIN_USER", "whisk_admin")
+ self.db_password = cfg.get("couchdb.admin.password", "COUCHDB_ADMIN_PASSWORD", "some_passw0rd")
+ self.db_auth = req.auth.HTTPBasicAuth(self.db_username,self.db_password)
+ self.db_url = f"{self.db_protocol}://{self.db_host}:{self.db_port}"
+ self.db_base = f"{self.db_url}/{self.db_prefix}"
+ self.db_session = req.Session()
+ self.db_session.auth = self.db_auth
+
+ def wait_db_ready(self, max_seconds):
+ logging.info("entering CouchDB.wait_db_ready()")
+ start = time.time()
+ delta = 0
+ session = req.Session()
+ while delta < max_seconds:
+ try:
+ r = session.get(f"{self.db_url}/_utils", timeout=5)
+ logging.info(f"CouchDB.wait_db_ready() got response code = {r.status_code}")
+ if r.status_code == 200:
+ return True
+ except Exception as e:
+ logging.info(f"waiting since: {delta} seconds")
+ delta = int(time.time() - start)
+ time.sleep(1)
+ return False
+
+ # check if database exists, return boolean
+ def check_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.head(url)
+ return r.status_code == 200
+
+ # delete database, return true if ok
+ def delete_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.delete(url)
+ return r.status_code == 200
+
+ # create db, return true if ok
+ def create_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.put(url)
+ return r.status_code == 201
+
+ # database="subjects"
+ def recreate_db(self, database, recreate=False):
+ msg = "recreate_db:"
+ exists = self.check_db(database)
+ if recreate and exists:
+ msg += " deleted"
+ self.delete_db(database)
+ if recreate or not exists:
+ msg += " created"
+ self.create_db(database)
+ return msg
+
+ def get_doc(self, database, id, user=None, password="", no_auth=False):
+ url = f"{self.db_base}{database}/{id}"
+ session = req.Session()
+ if no_auth:
+ db_auth=None
+ elif user:
+ db_auth=req.auth.HTTPBasicAuth(user, password)
+ else:
+ db_auth = self.db_auth
+ session.auth = db_auth
+ r = session.get(url)
+ if r.status_code == 200:
+ return json.loads(r.text)
+ return None
+
+ def update_doc(self, database, doc):
+ if '_id' in doc:
+ url = f"{self.db_base}{database}/{doc['_id']}"
+ cur = self.get_doc(database, doc['_id'])
+ if cur and '_rev' in cur:
+ doc['_rev'] = cur['_rev']
+ r = self.db_session.put(url, json=doc)
+ else:
+ r = self.db_session.put(url, json=doc)
+ return r.status_code in [200,201]
+ return False
+
+ def delete_doc(self, database, id):
+ cur = self.get_doc(database, id)
+ if cur and '_rev' in cur:
+ url = f"{self.db_base}{database}/{cur['_id']}?rev={cur['_rev']}"
+ r = self.db_session.delete(url)
+ return r.status_code == 200
+ return False
+
+ def configure_single_node(self):
+ url = f"{self.db_url}/_cluster_setup"
+ data = {"action": "enable_single_node", "singlenode": True, "bind_address": "0.0.0.0", "port": 5984}
+ r = self.db_session.post(url, json=data)
+ return r.status_code == 201
+
+ def configure_no_reduce_limit(self):
+ url = f"{self.db_url}/_node/_local/_config/query_server_config/reduce_limit"
+ data=b'"false"'
+ r = self.db_session.put(url, data=data)
+ return r.status_code == 200
+
+ def add_user(self, username: str, password: str):
+ userpass = {"name": username, "password": password, "roles": [], "type": "user"}
+ url = f"{self.db_url}/_users/org.couchdb.user:{username}"
+ res = self.db_session.put(url, json=userpass)
+ return res.status_code in [200, 201, 421]
+
+ #def add_role(self, database: str, members: list[str] = [], admins: list[str] =[]):
+ def add_role(self, database: str, members = [], admins =[]):
+ roles = {"admins": { "names": admins, "roles": [] }, "members": { "names": members, "roles": [] } }
+ url = f"{self.db_base}{database}/_security"
+ res = self.db_session.put(url, json=roles)
+ return res.status_code in [200, 201, 421]
+
+#
+# Submit a POST request to the _find endpoint using the specified selector
+#
+ def find_doc(self, database, selector, user=None, password="", no_auth=False):
+ url = f"{self.db_base}{database}/_find"
+ headers = {'Content-Type': 'application/json'}
+ session = req.Session()
+
+ if no_auth:
+ db_auth=None
+ elif user:
+ db_auth=req.auth.HTTPBasicAuth(user, password)
+ else:
+ db_auth = self.db_auth
+
+ session.auth=db_auth
+ r = session.post(url, headers=headers, data=selector)
+ if r.status_code == 200:
+ return json.loads(r.text)
+
+ logging.warn(f"query to {url} failed with {r.status_code}. Body {r.text}")
+ return None
\ No newline at end of file
diff --git a/actions/devel/ferretdb/__main__.py b/actions/devel/ferretdb/__main__.py
new file mode 100644
index 0000000..c741af8
--- /dev/null
+++ b/actions/devel/ferretdb/__main__.py
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+
+from common.authorize import Authorize
+from common.command_data import CommandData
+from command.ferretbd import FerretDB
+from base64 import b64decode
+
+class ApiError(Exception):
+ pass
+
+def build_error(message: str):
+ return {
+ "statusCode": 400,
+ "body": message
+ }
+
+def build_response(data:CommandData):
+ meta_data = data.get_metadata()
+ return {
+ "statusCode": meta_data['status'],
+ "body": meta_data['result']
+ }
+
+def parse_body(args):
+ try:
+ return b64decode(args['__ow_body']).decode().strip()
+ except Exception as e:
+ print(e)
+ raise ApiError("could not parse __ow_body as base64")
+
+def main(args):
+ """
+ Action implementing a generic command wrapper for the nuv devel mdb/ferretdb plugin. The action must be called with a POST request receiving a JSON
+ payload similar to this one
+ {
+ "command":"SET nuvolaris:key1 donald_duck"
+ }
+
+ the invoker must provide a x-impersonate-auth header containing the Openwhisk BASIC authentication of the wsku/user the action should impersonate
+ when submitting the command. Every specific provider will support specific request type and command.
+ WARNING: the body will be received as base64 encoded string as this action will be deployed with --web raw enabled flag
+ """
+ print(args)
+ headers = args['__ow_headers']
+ if('x-impersonate-auth' not in headers):
+ return build_error("invalid request, missing mandatory header: x-impersonate-auth")
+
+ if(len(args['__ow_body']) == 0):
+ return build_error("invalid request, no command payload received")
+
+ try:
+ user_data = Authorize(args['couchdb_host'],args['couchdb_user'],args['couchdb_password']).login(headers['x-impersonate-auth'])
+ cmd = CommandData(json.loads(parse_body(args)))
+ return build_response(FerretDB(user_data).execute(cmd))
+ except Exception as e:
+ return build_error(f"failed to execute nuv devel ferretdb. Reason: {str(e)}")
\ No newline at end of file
diff --git a/actions/devel/ferretdb/command/ferretbd.py b/actions/devel/ferretdb/command/ferretbd.py
new file mode 100644
index 0000000..42dbc0e
--- /dev/null
+++ b/actions/devel/ferretdb/command/ferretbd.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import common.util as ut
+import json
+
+
+from pymongo import MongoClient
+from common.command_data import CommandData
+
+class FerretDB():
+ """
+ Implementation of a FerretDB/Mongodb Command executor. It will require
+ a user_data dictionary linked to a specific user.
+ """
+
+ def __init__(self, user_data):
+ self._user_data = user_data
+ self._mdb_url = ut.get_env_value(user_data,"MONGODB_URL")
+ self._db = user_data['login']
+ self.validate()
+
+ def validate(self):
+ """
+ Validate that the provided user_data contains the appropriate
+ metadata for being able to submit a redis command.
+ """
+ if not self._mdb_url or not self._db:
+ raise Exception("user does not have valid FERRETDB/MONGODB environment set")
+
+ def _get_db(self):
+ """
+ Get a reference to the connected user database
+ """
+ client = MongoClient(self._mdb_url)
+ return client[self._db]
+
+
+ def _send_command(self,input:CommandData):
+ print(f"inside ferretdb _send_command {json.dumps(input.get_raw_data())}")
+ db = self._get_db()
+ response = db.command(input.get_raw_data())
+
+ if response:
+ input.result(str(response))
+ input.status(200)
+ else:
+ input.result(f"ferretdb/mongodb operation failed")
+ input.status(400)
+
+ def execute(self, input:CommandData):
+ try:
+ input.status(400)
+ self._send_command(input)
+ except Exception as e:
+ print(e)
+ input.result(f"could not execute ferretbd command {e}")
+ input.status(400)
+
+ return input
\ No newline at end of file
diff --git a/actions/devel/ferretdb/common/authorize.py b/actions/devel/ferretdb/common/authorize.py
new file mode 100644
index 0000000..f7d716f
--- /dev/null
+++ b/actions/devel/ferretdb/common/authorize.py
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from base64 import b64decode, b64encode
+from urllib.parse import quote, unquote
+
+import nuvolaris.config as cfg
+import nuvolaris.couchdb_util as cu
+import json
+import common.util as ut
+
+USER_META_DBN = "users_metadata"
+SUBJECT_META_DBN = "subjects"
+
+class DecodeError(Exception):
+ pass
+
+class EncodeError(Exception):
+ pass
+
+class AuthorizationError(Exception):
+ pass
+
+class Authorize():
+
+ def __init__(self, cdb_host, cdb_user, cdb_pwd):
+ cfg.clean()
+ cfg.put("couchdb.host", cdb_host)
+ cfg.put("couchdb.admin.user", cdb_user)
+ cfg.put("couchdb.admin.password", cdb_pwd)
+ self._db = cu.CouchDB()
+
+ def encode(self,username, password):
+ """Returns an HTTP basic authentication encrypted string given a valid
+ username and password.
+ """
+ if ':' in username:
+ raise EncodeError
+
+ username_password = f'{quote(username)}:{quote(password)}'
+ return f'Basic {b64encode(username_password.encode()).decode()}'
+
+ def _parse_b64(self, encoded_str):
+ try:
+ username, password = b64decode(encoded_str).decode().split(':', 1)
+ except:
+ # fallback in case the token is not bas64 encoded
+ username, password = encoded_str.split(':', 1)
+
+ if not username or not password:
+ raise DecodeError("authentication token does not seems to be b64 encoded")
+
+ return username, password
+
+ def decode(self, encoded_str):
+ """Decode an encrypted HTTP basic authentication string. Returns a tuple of
+ the form (username, password), and raises a DecodeError exception if
+ nothing could be decoded.
+ """
+ split = encoded_str.strip().split(' ')
+
+ # If split is only one element, try to decode the username and password
+ # directly.
+ if len(split) == 1:
+ username, password = self._parse_b64(split[0])
+
+ # If there are only two elements, check the first and ensure it says
+ # 'basic' so that we know we're about to decode the right thing. If not,
+ # bail out.
+ elif len(split) == 2:
+ if split[0].strip().lower() == 'basic':
+ username, password = self._parse_b64(split[1])
+ else:
+ raise DecodeError("authentication token provides more than 2 elements. could not parse properly")
+
+ # If there are more than 2 elements, something crazy must be happening.
+ # Bail.
+ else:
+ raise DecodeError("unpredictable error parsing authentication token")
+
+ return unquote(username), unquote(password)
+
+ def fetch_subject(self, uuid: str, key: str):
+ """
+ Query the internal couchdb searching for the subject mathing the given uuid, key.
+ Normally these stored in wsk or wsku in the form uuid:key
+ :param uuid the OW subject uuid
+ :param key the OW subject key
+ :return a ubject document
+ """
+ print(f"searching for openwhisk subject {uuid}")
+ try:
+ selector= {
+ "selector": {"namespaces": {"$elemMatch": {"uuid": uuid,"key": key}}}
+ }
+
+ response = self._db.find_doc(SUBJECT_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris namespace for user {uuid} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {uuid} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {uuid}. Reason: {e}")
+ return None
+
+ def fetch_user_data(self, username: str):
+ """
+ Query the internal couchdb searching for the given principal to retrieve all the
+ relevant metadata
+ """
+ print(f"searching for user {username} meta-data")
+ try:
+ selector = {"selector":{"login": {"$eq": username }}}
+ response = self._db.find_doc(USER_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris metadata for user {username} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {username} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {username}. Reason: {e}")
+ return None
+
+ def login(self, authorization: str):
+ """
+ Attempt to login the user identified by the given Openwhisk authorization AUTH token as base64
+ """
+ uuid, key = self.decode(authorization)
+ subject = self.fetch_subject(uuid, key)
+
+ if not subject:
+ raise AuthorizationError("Openwhisk subject not found.")
+
+ user_data = self.fetch_user_data(subject['subject'])
+
+ if user_data:
+ return user_data
+
+ raise AuthorizationError("Could not retrieve user metadata.")
\ No newline at end of file
diff --git a/actions/devel/ferretdb/common/command_data.py b/actions/devel/ferretdb/common/command_data.py
new file mode 100644
index 0000000..44f3faa
--- /dev/null
+++ b/actions/devel/ferretdb/common/command_data.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+import logging
+
+class CommandData:
+ _data = {}
+ _raw_data = {}
+
+ def __init__(self, cmd):
+ self._data = cmd
+ self._raw_data = cmd.copy()
+
+ def dump(self):
+ logging.debug(json.dumps(self._data))
+
+ def status(self, status):
+ self._data['status']=status
+
+ def result(self, result):
+ self._data['result']=result
+
+ def get_metadata(self):
+ return self._data
+
+ def get_raw_data(self):
+ return self._raw_data
+
+ def command(self):
+ return self._data['command']
+
+ def args(self):
+ return self._data['args']
+
+
diff --git a/actions/devel/ferretdb/common/minio_util.py b/actions/devel/ferretdb/common/minio_util.py
new file mode 100644
index 0000000..e1e8f90
--- /dev/null
+++ b/actions/devel/ferretdb/common/minio_util.py
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import base64
+import random
+import string
+import mimetypes
+
+from minio import Minio
+from minio.error import S3Error
+from minio.commonconfig import CopySource
+
+def extract_mimetype(file):
+ mimetype, _ = mimetypes.guess_type(file)
+
+ if mimetype is None:
+ return "application/octet-stream"
+ else:
+ return mimetype
+
+def build_mo_client(host, port, access_key, secret_key):
+ """
+ Creates an Minio client pointing to the given MINIO HOST
+ :param host, minio host
+ :param port, minio api port normally it is the 9000
+ :param access_key user we are representing
+ :param secret_key to access minio
+ """
+ mo_client = Minio(f"{host}:{port}",access_key=access_key,secret_key=secret_key,secure=False)
+ return mo_client
+
+def upload_file(mo_client, file, bucket, object_name=None):
+ """Upload a file to an S3 bucket
+
+ :param file_name: File to upload
+ :param bucket: Bucket to upload to
+ :param object_name: S3 object name. If not specified then file_name is used
+ :return: a tuple with the Booleand str,
+ """
+
+ # If S3 object_name was not specified, use file_name
+ if object_name is None:
+ object_name = os.path.basename(file)
+
+ print(f"uploading {object_name} into bucket {bucket} from tmp_file {file}")
+
+ # Upload the file
+ try:
+ mimetype = extract_mimetype(object_name)
+ response = mo_client.fput_object(bucket_name=bucket,object_name=object_name,file_path=file,content_type=mimetype)
+ if response._object_name:
+ return True, response._object_name
+ except Exception as e:
+ print(e)
+ return False, str(e)
+ return False, "KO"
+
+def prepare_file_upload(username, filename, file_content_as_b64):
+ """ Creates a tmp area for the given user where the uploaded file will be stored under a random generated name
+ the fully qualified filename is taken into account only on the corresponding destination bucket.
+ param: username
+ param: filename
+ param: file_content_as_b64
+ return: a file object pointing to the tmp file
+ """
+ try:
+ user_tmp_folder = f"/tmp/{username}"
+ if not os.path.exists(user_tmp_folder):
+ os.makedirs(user_tmp_folder)
+ print(f"added tmp folder {user_tmp_folder}")
+
+ delete_files_in_directory_and_subdirectories(user_tmp_folder)
+ rnd_filename = get_random_string(20)
+ tmp_file = f"{user_tmp_folder}/{rnd_filename}"
+
+ with open(tmp_file, "wb") as f:
+ file_content=base64.b64decode(file_content_as_b64)
+ f.write(file_content)
+
+ if os.path.exists(tmp_file):
+ print(f"{tmp_file} stored successfully.")
+ return tmp_file
+ else:
+ return None
+ except Exception as e:
+ print("error preparing tmp_files",e)
+ return None
+
+def delete_files_in_directory_and_subdirectories(directory_path):
+ try:
+ for root, dirs, files in os.walk(directory_path):
+ for file in files:
+ file_path = os.path.join(root, file)
+ os.remove(file_path)
+ print("All files and subdirectories deleted successfully.")
+ except OSError:
+ print("Error occurred while deleting files and subdirectories.")
+
+def get_random_string(length):
+ # choose from all lowercase letter
+ letters = string.ascii_lowercase
+ return ''.join(random.choice(letters) for i in range(length))
+
+def rm_file(mo_client, bucket, file):
+ """ Remove a file from a bucket
+ :parma mo_client a minio client instance to execute the command
+ :param bucket the bucket name
+ :param file the file name
+ :return True if the file has been removed, False otherwise
+ """
+
+ try:
+ mo_client.remove_object(bucket, file)
+ return True
+ except Exception as e:
+ print(e)
+ return False
+ return False
+
+def cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Copy a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+ try:
+ # copy an object from a bucket to another.
+ cp_result = mo_client.copy_object(
+ dest_bucket,
+ dest_file,
+ CopySource(orig_bucket, orig_file)
+ )
+ if cp_result.object_name:
+ return cp_result.object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
+def mv_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Move a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+
+ # Upload the file
+ try:
+ # mv an object from a bucket to another.
+ object_name = cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file)
+ if object_name:
+ rm_file(mo_client,orig_bucket,orig_file)
+ return object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
diff --git a/actions/devel/ferretdb/common/util.py b/actions/devel/ferretdb/common/util.py
new file mode 100644
index 0000000..f1e8f39
--- /dev/null
+++ b/actions/devel/ferretdb/common/util.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import mimetypes
+
+def get_env_value(user_data, key):
+ """
+ Check if inside the given user_data object there is an env item with the
+ given name and returns the value if any.
+ :param user_data
+ :param key
+ :return None if the given key it is not prese
+ """
+ envs = list(user_data['env'])
+
+ for env in envs:
+ if env['key'] == key:
+ return env['value']
+
+ return None
\ No newline at end of file
diff --git a/actions/devel/ferretdb/flatdict.py b/actions/devel/ferretdb/flatdict.py
new file mode 100644
index 0000000..834ce99
--- /dev/null
+++ b/actions/devel/ferretdb/flatdict.py
@@ -0,0 +1,506 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""FlatDict is a dict object that allows for single level, delimited
+key/value pair mapping of nested dictionaries.
+
+"""
+try:
+ from collections.abc import MutableMapping
+except ImportError: # pragma: nocover
+ from collections import MutableMapping
+import sys
+
+__version__ = '4.0.1'
+
+NO_DEFAULT = object()
+
+
+class FlatDict(MutableMapping):
+ """:class:`~flatdict.FlatDict` is a dictionary object that allows for
+ single level, delimited key/value pair mapping of nested dictionaries.
+ The default delimiter value is ``:`` but can be changed in the constructor
+ or by calling :meth:`FlatDict.set_delimiter`.
+
+ """
+ _COERCE = dict
+
+ def __init__(self, value=None, delimiter=':', dict_class=dict):
+ super(FlatDict, self).__init__()
+ self._values = dict_class()
+ self._delimiter = delimiter
+ self.update(value)
+
+ def __contains__(self, key):
+ """Check to see if the key exists, checking for both delimited and
+ not delimited key values.
+
+ :param mixed key: The key to check for
+
+ """
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ return pk in self._values and ck in self._values[pk]
+ return key in self._values
+
+ def __delitem__(self, key):
+ """Delete the item for the specified key, automatically dealing with
+ nested children.
+
+ :param mixed key: The key to use
+ :raises: KeyError
+
+ """
+ if key not in self:
+ raise KeyError
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ del self._values[pk][ck]
+ if not self._values[pk]:
+ del self._values[pk]
+ else:
+ del self._values[key]
+
+ def __eq__(self, other):
+ """Check for equality against the other value
+
+ :param other: The value to compare
+ :type other: FlatDict
+ :rtype: bool
+ :raises: TypeError
+
+ """
+ if isinstance(other, dict):
+ return self.as_dict() == other
+ elif not isinstance(other, self.__class__):
+ raise TypeError
+ return self.as_dict() == other.as_dict()
+
+ def __ne__(self, other):
+ """Check for inequality against the other value
+
+ :param other: The value to compare
+ :type other: dict or FlatDict
+ :rtype: bool
+
+ """
+ return not self.__eq__(other)
+
+ def __getitem__(self, key):
+ """Get an item for the specified key, automatically dealing with
+ nested children.
+
+ :param mixed key: The key to use
+ :rtype: mixed
+ :raises: KeyError
+
+ """
+ values = self._values
+ key = [key] if isinstance(key, int) else key.split(self._delimiter)
+ for part in key:
+ values = values[part]
+ return values
+
+ def __iter__(self):
+ """Iterate over the flat dictionary key and values
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ return iter(self.keys())
+
+ def __len__(self):
+ """Return the number of items.
+
+ :rtype: int
+
+ """
+ return len(self.keys())
+
+ def __reduce__(self):
+ """Return state information for pickling
+
+ :rtype: tuple
+
+ """
+ return type(self), (self.as_dict(), self._delimiter)
+
+ def __repr__(self):
+ """Return the string representation of the instance.
+
+ :rtype: str
+
+ """
+ return '<{} id={} {}>"'.format(self.__class__.__name__, id(self),
+ str(self))
+
+ def __setitem__(self, key, value):
+ """Assign the value to the key, dynamically building nested
+ FlatDict items where appropriate.
+
+ :param mixed key: The key for the item
+ :param mixed value: The value for the item
+ :raises: TypeError
+
+ """
+ if isinstance(value, self._COERCE) and not isinstance(value, FlatDict):
+ value = self.__class__(value, self._delimiter)
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if pk not in self._values:
+ self._values[pk] = self.__class__({ck: value}, self._delimiter)
+ return
+ elif not isinstance(self._values[pk], FlatDict):
+ raise TypeError(
+ 'Assignment to invalid type for key {}'.format(pk))
+ self._values[pk][ck] = value
+ else:
+ self._values[key] = value
+
+ def __str__(self):
+ """Return the string value of the instance.
+
+ :rtype: str
+
+ """
+ return '{{{}}}'.format(', '.join(
+ ['{!r}: {!r}'.format(k, self[k]) for k in self.keys()]))
+
+ def as_dict(self):
+ """Return the :class:`~flatdict.FlatDict` as a :class:`dict`
+
+ :rtype: dict
+
+ """
+ out = dict({})
+ for key in self.keys():
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if self._has_delimiter(ck):
+ ck = ck.split(self._delimiter, 1)[0]
+ if isinstance(self._values[pk], FlatDict) and pk not in out:
+ out[pk] = {}
+ if isinstance(self._values[pk][ck], FlatDict):
+ out[pk][ck] = self._values[pk][ck].as_dict()
+ else:
+ out[pk][ck] = self._values[pk][ck]
+ else:
+ out[key] = self._values[key]
+ return out
+
+ def clear(self):
+ """Remove all items from the flat dictionary."""
+ self._values.clear()
+
+ def copy(self):
+ """Return a shallow copy of the flat dictionary.
+
+ :rtype: flatdict.FlatDict
+
+ """
+ return self.__class__(self.as_dict(), delimiter=self._delimiter)
+
+ def get(self, key, d=None):
+ """Return the value for key if key is in the flat dictionary, else
+ default. If default is not given, it defaults to ``None``, so that this
+ method never raises :exc:`KeyError`.
+
+ :param mixed key: The key to get
+ :param mixed d: The default value
+ :rtype: mixed
+
+ """
+ try:
+ return self.__getitem__(key)
+ except KeyError:
+ return d
+
+ def items(self):
+ """Return a copy of the flat dictionary's list of ``(key, value)``
+ pairs.
+
+ .. note:: CPython implementation detail: Keys and values are listed in
+ an arbitrary order which is non-random, varies across Python
+ implementations, and depends on the flat dictionary's history of
+ insertions and deletions.
+
+ :rtype: list
+
+ """
+ return [(k, self.__getitem__(k)) for k in self.keys()]
+
+ def iteritems(self):
+ """Return an iterator over the flat dictionary's (key, value) pairs.
+ See the note for :meth:`flatdict.FlatDict.items`.
+
+ Using ``iteritems()`` while adding or deleting entries in the flat
+ dictionary may raise :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for item in self.items():
+ yield item
+
+ def iterkeys(self):
+ """Iterate over the flat dictionary's keys. See the note for
+ :meth:`flatdict.FlatDict.items`.
+
+ Using ``iterkeys()`` while adding or deleting entries in the flat
+ dictionary may raise :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for key in self.keys():
+ yield key
+
+ def itervalues(self):
+ """Return an iterator over the flat dictionary's values. See the note
+ :meth:`flatdict.FlatDict.items`.
+
+ Using ``itervalues()`` while adding or deleting entries in the flat
+ dictionary may raise a :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for value in self.values():
+ yield value
+
+ def keys(self):
+ """Return a copy of the flat dictionary's list of keys.
+ See the note for :meth:`flatdict.FlatDict.items`.
+
+ :rtype: list
+
+ """
+ keys = []
+
+ for key, value in self._values.items():
+ if isinstance(value, (FlatDict, dict)):
+ nested = [
+ self._delimiter.join([str(key), str(k)])
+ for k in value.keys()]
+ keys += nested if nested else [key]
+ else:
+ keys.append(key)
+
+ return keys
+
+ def pop(self, key, default=NO_DEFAULT):
+ """If key is in the flat dictionary, remove it and return its value,
+ else return default. If default is not given and key is not in the
+ dictionary, :exc:`KeyError` is raised.
+
+ :param mixed key: The key name
+ :param mixed default: The default value
+ :rtype: mixed
+
+ """
+ if key not in self and default != NO_DEFAULT:
+ return default
+ value = self[key]
+ self.__delitem__(key)
+ return value
+
+ def setdefault(self, key, default):
+ """If key is in the flat dictionary, return its value. If not,
+ insert key with a value of default and return default.
+ default defaults to ``None``.
+
+ :param mixed key: The key name
+ :param mixed default: The default value
+ :rtype: mixed
+
+ """
+ if key not in self:
+ self.__setitem__(key, default)
+ return self.__getitem__(key)
+
+ def set_delimiter(self, delimiter):
+ """Override the default or passed in delimiter with a new value. If
+ the requested delimiter already exists in a key, a :exc:`ValueError`
+ will be raised.
+
+ :param str delimiter: The delimiter to use
+ :raises: ValueError
+
+ """
+ for key in self.keys():
+ if delimiter in key:
+ raise ValueError('Key {!r} collides with delimiter {!r}', key,
+ delimiter)
+ self._delimiter = delimiter
+ for key in self._values.keys():
+ if isinstance(self._values[key], FlatDict):
+ self._values[key].set_delimiter(delimiter)
+
+ def update(self, other=None, **kwargs):
+ """Update the flat dictionary with the key/value pairs from other,
+ overwriting existing keys.
+
+ ``update()`` accepts either another flat dictionary object or an
+ iterable of key/value pairs (as tuples or other iterables of length
+ two). If keyword arguments are specified, the flat dictionary is then
+ updated with those key/value pairs: ``d.update(red=1, blue=2)``.
+
+ :param iterable other: Iterable of key, value pairs
+ :rtype: None
+
+ """
+ [self.__setitem__(k, v) for k, v in dict(other or kwargs).items()]
+
+ def values(self):
+ """Return a copy of the flat dictionary's list of values. See the note
+ for :meth:`flatdict.FlatDict.items`.
+
+ :rtype: list
+
+ """
+ return [self.__getitem__(k) for k in self.keys()]
+
+ def _has_delimiter(self, key):
+ """Checks to see if the key contains the delimiter.
+
+ :rtype: bool
+
+ """
+ return isinstance(key, str) and self._delimiter in key
+
+
+class FlatterDict(FlatDict):
+ """Like :class:`~flatdict.FlatDict` but also coerces lists and sets
+ to child-dict instances with the offset as the key. Alternative to
+ the implementation added in v1.2 of FlatDict.
+
+ """
+ _COERCE = list, tuple, set, dict, FlatDict
+ _ARRAYS = list, set, tuple
+
+ def __init__(self, value=None, delimiter=':', dict_class=dict):
+ self.original_type = type(value)
+ if self.original_type in self._ARRAYS:
+ value = {str(i): v for i, v in enumerate(value)}
+ super(FlatterDict, self).__init__(value, delimiter, dict_class)
+
+ def __setitem__(self, key, value):
+ """Assign the value to the key, dynamically building nested
+ FlatDict items where appropriate.
+
+ :param mixed key: The key for the item
+ :param mixed value: The value for the item
+ :raises: TypeError
+
+ """
+ if isinstance(value, self._COERCE) and \
+ not isinstance(value, FlatterDict):
+ value = self.__class__(value, self._delimiter)
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if pk not in self._values:
+ self._values[pk] = self.__class__({ck: value}, self._delimiter)
+ return
+ if getattr(self._values[pk], 'original_type',
+ None) in self._ARRAYS:
+ try:
+ k, cck = ck.split(self._delimiter, 1)
+ int(k)
+ except ValueError:
+ raise TypeError(
+ 'Assignment to invalid type for key {}{}{}'.format(
+ pk, self._delimiter, ck))
+ self._values[pk][k][cck] = value
+ return
+ elif not isinstance(self._values[pk], FlatterDict):
+ raise TypeError(
+ 'Assignment to invalid type for key {}'.format(pk))
+ self._values[pk][ck] = value
+ else:
+ self._values[key] = value
+
+ def as_dict(self):
+ """Return the :class:`~flatdict.FlatterDict` as a nested
+ :class:`dict`.
+
+ :rtype: dict
+
+ """
+ out = {}
+ for key in self.keys():
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if self._has_delimiter(ck):
+ ck = ck.split(self._delimiter, 1)[0]
+ if isinstance(self._values[pk], FlatterDict) and pk not in out:
+ if self._values[pk].original_type == tuple:
+ out[pk] = tuple(self._child_as_list(pk))
+ elif self._values[pk].original_type == list:
+ out[pk] = self._child_as_list(pk)
+ elif self._values[pk].original_type == set:
+ out[pk] = set(self._child_as_list(pk))
+ elif self._values[pk].original_type == dict:
+ out[pk] = self._values[pk].as_dict()
+ else:
+ if isinstance(self._values[key], FlatterDict):
+ out[key] = self._values[key].original_type()
+ else:
+ out[key] = self._values[key]
+ return out
+
+ def _child_as_list(self, pk, ck=None):
+ """Returns a list of values from the child FlatterDict instance
+ with string based integer keys.
+
+ :param str pk: The parent key
+ :param str ck: The child key, optional
+ :rtype: list
+
+ """
+ if ck is None:
+ subset = self._values[pk]
+ else:
+ subset = self._values[pk][ck]
+ # Check if keys has delimiter, which implies deeply nested dict
+ keys = subset.keys()
+ if any(self._has_delimiter(k) for k in keys):
+ out = []
+ split_keys = {k.split(self._delimiter)[0] for k in keys}
+ for k in sorted(split_keys, key=lambda x: int(x)):
+ if subset[k].original_type == tuple:
+ out.append(tuple(self._child_as_list(pk, k)))
+ elif subset[k].original_type == list:
+ out.append(self._child_as_list(pk, k))
+ elif subset[k].original_type == set:
+ out.append(set(self._child_as_list(pk, k)))
+ elif subset[k].original_type == dict:
+ out.append(subset[k].as_dict())
+ return out
+
+ # Python prior 3.6 does not guarantee insertion order, remove it after
+ # EOL python 3.5 - 2020-09-13
+ if sys.version_info[0:2] < (3, 6): # pragma: nocover
+ return [subset[k] for k in sorted(keys, key=lambda x: int(x))]
+ else:
+ return [subset[k] for k in keys]
diff --git a/actions/devel/ferretdb/nuvolaris/config.py b/actions/devel/ferretdb/nuvolaris/config.py
new file mode 100644
index 0000000..787d4a7
--- /dev/null
+++ b/actions/devel/ferretdb/nuvolaris/config.py
@@ -0,0 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import flatdict, json, os
+import logging
+
+_config = {}
+
+# define a configuration
+# the configuration is a map, followed by a list of labels
+# the map can be a serialized json and will be flattened to a map of values.
+# you can have only a configuration active at a time
+# if you want to set a new configuration you have to clean it
+def configure(spec: dict):
+ global _config
+ _config = dict(flatdict.FlatDict(spec, delimiter="."))
+
+ #forces autodetect of kube if not provided
+ if not exists('nuvolaris.kube'):
+ put('nuvolaris.kube','auto')
+
+ return True
+
+def clean():
+ global _config
+ _config = {}
+
+def exists(key):
+ return key in _config
+
+def get(key, envvar=None, defval=None):
+ val = _config.get(key)
+
+ if envvar and envvar in os.environ:
+ val = os.environ[envvar]
+
+ if val:
+ return val
+
+ return defval
+
+def put(key, value):
+ _config[key] = value
+ return True
+
+def delete(key):
+ if key in _config:
+ del _config[key]
+ return True
+
+def getall(prefix=""):
+ res = {}
+ for key in _config.keys():
+ if key.startswith(prefix):
+ res[key] = _config[key]
+ return res
+
+def keys(prefix=""):
+ res = []
+ if _config:
+ for key in _config.keys():
+ if key.startswith(prefix):
+ res.append(key)
+ return res
+
+def detect_labels(labels=None):
+ # skip autodetection of nuvolaris.kube if already available in _config
+ if exists('nuvolaris.kube') and get('nuvolaris.kube') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.kube={get('nuvolaris.kube')}")
+ return {}
+
+ # read labels if not available
+ if not labels:
+ import nuvolaris.kube as kube
+ labels = kube.kubectl("get", "nodes", jsonpath='{.items[].metadata.labels}')
+
+ res = {}
+ kube = None
+ for i in labels:
+ for j in list(i.keys()):
+ if j.startswith("nuvolaris.io/"):
+ key = f"nuvolaris.{j[13:]}"
+ res[key] = i[j]
+ _config[key] = i[j]
+
+ for i in labels:
+ for j in list(i.keys()):
+ # detect the kube type
+ if j.find("eksctl.io") >= 0:
+ kube ="eks"
+ break
+ if j.find("k8s.io/cloud-provider-aws") >= 0:
+ kube ="eks"
+ break
+ elif j.find("microk8s.io") >= 0:
+ kube = "microk8s"
+ break
+ elif j.find("lke.linode.com") >=0:
+ kube = "lks"
+ break
+ elif j.find("node.openshift.io") >=0:
+ kube = "openshift"
+ break
+ elif j.endswith("kubernetes.io/instance-type") and i[j] == "k3s":
+ kube = "k3s"
+ break
+ elif j.find("cloud.google.com/gke") >=0:
+ kube = "gke"
+ break
+ elif j.find("kubernetes.azure.com") >=0:
+ kube = "aks"
+ break
+ # assign all the 'nuvolaris.io' labels
+
+ if kube:
+ res["nuvolaris.kube"] = kube
+ _config["nuvolaris.kube"] = kube
+ return res
+
+ # defaults to generic if it is not yet detected
+ if exists('nuvolaris.kube') and get('nuvolaris.kube') == 'auto':
+ _config["nuvolaris.kube"] = "generic"
+ res["nuvolaris.kube"] = "generic"
+
+ return res
+
+def detect_storage(storages=None):
+ res = {}
+ if not storages:
+ import nuvolaris.util as util
+
+ try:
+ detect_storageclass = True
+ detect_provisioner = True
+
+ # skips autodetection if already provided
+ if exists('nuvolaris.storageclass') and get('nuvolaris.storageclass') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.storageclass={get('nuvolaris.storageclass')}")
+ detect_storageclass = False
+
+ if exists('nuvolaris.provisioner') and get('nuvolaris.provisioner') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.provisioner={get('nuvolaris.provisioner')}")
+ detect_provisioner = False
+
+ if not detect_storageclass and not detect_provisioner:
+ return res
+
+ if detect_storageclass:
+ storage_class = util.get_default_storage_class()
+ if storage_class:
+ res['nuvolaris.storageclass'] = storage_class
+ _config['nuvolaris.storageclass'] = storage_class
+
+ if detect_provisioner:
+ provisioner = util.get_default_storage_provisioner()
+ if(provisioner):
+ res['nuvolaris.provisioner'] = provisioner
+ _config['nuvolaris.provisioner'] = provisioner
+ except:
+ pass
+ return res
+
+def detect_env():
+ _config['operator.image'] = os.environ.get("OPERATOR_IMAGE", "missing-OPERATOR_IMAGE")
+ _config['operator.tag'] = os.environ.get("OPERATOR_TAG", "missing-OPERATOR_TAG")
+
+ # skip autodetection of controller.image if already configure into whisk.yaml
+ if not exists('controller.image'):
+ _config['controller.image'] = os.environ.get("CONTROLLER_IMAGE", "missing-CONTROLLER_IMAGE")
+ _config['controller.tag'] = os.environ.get("CONTROLLER_TAG", "missing-CONTROLLER_TAG")
+ else:
+ logging.warn(f"OW controller image detection skipped. Using {get('controller.image')}")
+
+ if not exists('invoker.image'):
+ _config['invoker.image'] = os.environ.get("INVOKER_IMAGE", "missing-INVOKER_IMAGE")
+ _config['invoker.tag'] = os.environ.get("INVOKER_TAG", "missing-INVOKER_TAG")
+ else:
+ logging.warn(f"OW invoker image detection skipped. Using {get('invoker.image')}")
+
+def detect():
+ detect_storage()
+ detect_labels()
+ detect_env()
+
+def dump_config():
+ import nuvolaris.config as cfg
+ for k in cfg.getall():
+ logging.debug(f"{k} = {cfg.get(k)}")
diff --git a/actions/devel/ferretdb/nuvolaris/couchdb_util.py b/actions/devel/ferretdb/nuvolaris/couchdb_util.py
new file mode 100644
index 0000000..3adb6b9
--- /dev/null
+++ b/actions/devel/ferretdb/nuvolaris/couchdb_util.py
@@ -0,0 +1,164 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import os, json, time, sys, logging
+import requests as req
+import nuvolaris.config as cfg
+
+class CouchDB:
+ def __init__(self):
+ self.db_protocol = "http"
+ self.db_prefix = "nuvolaris_"
+ self.db_port = cfg.get("couchdb.port", "COUCHDB_SERVICE_PORT", "5984")
+ self.db_host = cfg.get("couchdb.host", "COUCHDB_SERVICE_HOST", "couchdb")
+ self.db_username = cfg.get("couchdb.admin.user", "COUCHDB_ADMIN_USER", "whisk_admin")
+ self.db_password = cfg.get("couchdb.admin.password", "COUCHDB_ADMIN_PASSWORD", "some_passw0rd")
+ self.db_auth = req.auth.HTTPBasicAuth(self.db_username,self.db_password)
+ self.db_url = f"{self.db_protocol}://{self.db_host}:{self.db_port}"
+ self.db_base = f"{self.db_url}/{self.db_prefix}"
+ self.db_session = req.Session()
+ self.db_session.auth = self.db_auth
+
+ def wait_db_ready(self, max_seconds):
+ logging.info("entering CouchDB.wait_db_ready()")
+ start = time.time()
+ delta = 0
+ session = req.Session()
+ while delta < max_seconds:
+ try:
+ r = session.get(f"{self.db_url}/_utils", timeout=5)
+ logging.info(f"CouchDB.wait_db_ready() got response code = {r.status_code}")
+ if r.status_code == 200:
+ return True
+ except Exception as e:
+ logging.info(f"waiting since: {delta} seconds")
+ delta = int(time.time() - start)
+ time.sleep(1)
+ return False
+
+ # check if database exists, return boolean
+ def check_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.head(url)
+ return r.status_code == 200
+
+ # delete database, return true if ok
+ def delete_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.delete(url)
+ return r.status_code == 200
+
+ # create db, return true if ok
+ def create_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.put(url)
+ return r.status_code == 201
+
+ # database="subjects"
+ def recreate_db(self, database, recreate=False):
+ msg = "recreate_db:"
+ exists = self.check_db(database)
+ if recreate and exists:
+ msg += " deleted"
+ self.delete_db(database)
+ if recreate or not exists:
+ msg += " created"
+ self.create_db(database)
+ return msg
+
+ def get_doc(self, database, id, user=None, password="", no_auth=False):
+ url = f"{self.db_base}{database}/{id}"
+ session = req.Session()
+ if no_auth:
+ db_auth=None
+ elif user:
+ db_auth=req.auth.HTTPBasicAuth(user, password)
+ else:
+ db_auth = self.db_auth
+ session.auth = db_auth
+ r = session.get(url)
+ if r.status_code == 200:
+ return json.loads(r.text)
+ return None
+
+ def update_doc(self, database, doc):
+ if '_id' in doc:
+ url = f"{self.db_base}{database}/{doc['_id']}"
+ cur = self.get_doc(database, doc['_id'])
+ if cur and '_rev' in cur:
+ doc['_rev'] = cur['_rev']
+ r = self.db_session.put(url, json=doc)
+ else:
+ r = self.db_session.put(url, json=doc)
+ return r.status_code in [200,201]
+ return False
+
+ def delete_doc(self, database, id):
+ cur = self.get_doc(database, id)
+ if cur and '_rev' in cur:
+ url = f"{self.db_base}{database}/{cur['_id']}?rev={cur['_rev']}"
+ r = self.db_session.delete(url)
+ return r.status_code == 200
+ return False
+
+ def configure_single_node(self):
+ url = f"{self.db_url}/_cluster_setup"
+ data = {"action": "enable_single_node", "singlenode": True, "bind_address": "0.0.0.0", "port": 5984}
+ r = self.db_session.post(url, json=data)
+ return r.status_code == 201
+
+ def configure_no_reduce_limit(self):
+ url = f"{self.db_url}/_node/_local/_config/query_server_config/reduce_limit"
+ data=b'"false"'
+ r = self.db_session.put(url, data=data)
+ return r.status_code == 200
+
+ def add_user(self, username: str, password: str):
+ userpass = {"name": username, "password": password, "roles": [], "type": "user"}
+ url = f"{self.db_url}/_users/org.couchdb.user:{username}"
+ res = self.db_session.put(url, json=userpass)
+ return res.status_code in [200, 201, 421]
+
+ #def add_role(self, database: str, members: list[str] = [], admins: list[str] =[]):
+ def add_role(self, database: str, members = [], admins =[]):
+ roles = {"admins": { "names": admins, "roles": [] }, "members": { "names": members, "roles": [] } }
+ url = f"{self.db_base}{database}/_security"
+ res = self.db_session.put(url, json=roles)
+ return res.status_code in [200, 201, 421]
+
+#
+# Submit a POST request to the _find endpoint using the specified selector
+#
+ def find_doc(self, database, selector, user=None, password="", no_auth=False):
+ url = f"{self.db_base}{database}/_find"
+ headers = {'Content-Type': 'application/json'}
+ session = req.Session()
+
+ if no_auth:
+ db_auth=None
+ elif user:
+ db_auth=req.auth.HTTPBasicAuth(user, password)
+ else:
+ db_auth = self.db_auth
+
+ session.auth=db_auth
+ r = session.post(url, headers=headers, data=selector)
+ if r.status_code == 200:
+ return json.loads(r.text)
+
+ logging.warn(f"query to {url} failed with {r.status_code}. Body {r.text}")
+ return None
\ No newline at end of file
diff --git a/actions/devel/minio/__main__.py b/actions/devel/minio/__main__.py
new file mode 100644
index 0000000..14f37df
--- /dev/null
+++ b/actions/devel/minio/__main__.py
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import nuvolaris.config as cfg
+import nuvolaris.couchdb_util as cu
+import logging, json
+
+from common.authorize import Authorize
+from command.minio import Minio
+from common.command_data import CommandData
+from base64 import b64decode
+
+class ApiError(Exception):
+ pass
+
+def build_error(message: str):
+ return {
+ "statusCode": 400,
+ "body": message
+ }
+
+def build_response(data:CommandData):
+ meta_data = data.get_metadata()
+ return {
+ "statusCode": meta_data['status'],
+ "body": meta_data['result']
+ }
+
+def parse_body(args):
+ try:
+ return b64decode(args['__ow_body']).decode().strip()
+ except Exception as e:
+ print(e)
+ raise ApiError("could not parse __ow_body as base64")
+
+def main(args):
+ """
+ Action implementing a generic command wrapper for the nuv devel plugin. The action must be called with a POST request receiving a JSON
+ payload similar to this one
+ {
+ "command":"ls",
+ "args":[
+ "<bucket-name>"
+ ]
+ }
+
+ the invoker must provide a x-impersonate-auth header containing the Openwhisk BASIC authentication of the wsku/user the action should impersonate
+ when submitting the command. Every specific provider will support specific request type and command.
+ WARNING: the body will be received as base64 encoded string as this action will be deployed with --web raw enabled flag
+ """
+ headers = args['__ow_headers']
+ if('x-impersonate-auth' not in headers):
+ return build_error("invalid request, missing mandatory header: x-impersonate-auth")
+
+ if(len(args['__ow_body']) == 0):
+ return build_error("invalid request, no command payload received")
+
+ try:
+ user_data = Authorize(args['couchdb_host'],args['couchdb_user'],args['couchdb_password']).login(headers['x-impersonate-auth'])
+ cmd = CommandData(json.loads(parse_body(args)))
+ return build_response(Minio(user_data).execute(cmd))
+ except Exception as e:
+ return build_error(f"failed to execute nuv devel minio. Reason: {str(e)}")
\ No newline at end of file
diff --git a/actions/devel/minio/command/minio.py b/actions/devel/minio/command/minio.py
new file mode 100644
index 0000000..252f375
--- /dev/null
+++ b/actions/devel/minio/command/minio.py
@@ -0,0 +1,170 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import common.util as ut
+import common.minio_util as mutil
+import json
+import re
+
+from common.command_data import CommandData
+
+from minio.commonconfig import CopySource
+
+class Minio():
+ """
+ Implementation of a Minio Command executor. It will require
+ a user_data dictionary linked to a specific user.
+ """
+
+ def __init__(self, user_data):
+ self._user_data = user_data
+ self._minio_access_key= ut.get_env_value(user_data,"MINIO_ACCESS_KEY")
+ self._minio_secret_key= ut.get_env_value(user_data,"MINIO_SECRET_KEY")
+ self._minio_host= ut.get_env_value(user_data,"MINIO_HOST")
+ self._minio_port= ut.get_env_value(user_data,"MINIO_PORT")
+ self._minio_data_bucket= ut.get_env_value(user_data,"MINIO_DATA_BUCKET")
+ self._minio_static_bucket= ut.get_env_value(user_data,"MINIO_STATIC_BUCKET")
+ self.validate()
+
+ def validate(self):
+ """
+ Validate that the provided user_data contains the appropriate
+ metadata for being able to submit a postgres command.
+ """
+ if not self._minio_access_key or not self._minio_secret_key or not self._minio_host or not self._minio_port:
+ raise Exception("user does not have valid MINIO environment set")
+
+ def _list_buckets(self, input:CommandData):
+ print("**** listing user buckets")
+ mo_client = mutil.build_mo_client(self._minio_host, self._minio_port,self._minio_access_key, self._minio_secret_key)
+ result = []
+
+ buckets = mo_client.list_buckets()
+
+ for bucket in buckets:
+ result.append({"bucket": bucket.name, "creation_date": str(bucket.creation_date)})
+
+ input.result(result)
+ input.status(200)
+
+ def _list_bucket_content(self,bucket,input:CommandData):
+ print(f"**** listing bucket {bucket} content")
+
+ mo_client = mutil.build_mo_client(self._minio_host, self._minio_port,self._minio_access_key, self._minio_secret_key)
+ result = []
+ objects = mo_client.list_objects(bucket_name=bucket, recursive= True)
+
+ for obj in objects:
+ result.append({"name":obj.object_name,"last_modified": str(obj.last_modified), "size":obj.size})
+
+ input.result(result)
+ input.status(200)
+
+ def _rm_bucket_object(self,bucket,filename,input:CommandData):
+ print(f"**** removing file {filename} inside bucket {bucket}")
+
+ mo_client = mutil.build_mo_client(self._minio_host, self._minio_port,self._minio_access_key, self._minio_secret_key)
+ mo_client.remove_object(bucket, filename)
+
+ result = {"result":"OK"}
+ input.result(result)
+ input.status(200)
+
+ def _mv_bucket_object(self,orig_bucket,orig_filename,dest_bucket,dest_filename,input:CommandData):
+ print(f"**** moving file {orig_filename} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_filename}")
+
+ mo_client = mutil.build_mo_client(self._minio_host, self._minio_port,self._minio_access_key, self._minio_secret_key)
+ object_name = mutil.mv_file(mo_client,orig_bucket,orig_filename,dest_bucket,dest_filename)
+
+ if object_name:
+ input.result({"name":object_name})
+ input.status(200)
+ else:
+ input.result(f"move operation failed")
+ input.status(400)
+
+ def _cp_bucket_object(self,orig_bucket,orig_filename,dest_bucket,dest_filename,input:CommandData):
+ print(f"**** copying file {orig_filename} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_filename}")
+
+ mo_client = mutil.build_mo_client(self._minio_host, self._minio_port,self._minio_access_key, self._minio_secret_key)
+ object_name = mutil.cp_file(mo_client,orig_bucket,orig_filename,dest_bucket,dest_filename)
+
+ if object_name:
+ input.result({"name":object_name})
+ input.status(200)
+ else:
+ input.result(f"move operation failed")
+ input.status(400)
+
+ def _clean_bucket_content(self,input:CommandData, bucket, pattern=".*", dry_run="true"):
+ """"
+ Removes matching content from the specified bucket
+ :param input the entire input command
+ :param bucket
+ :param pattern a regexp specifying the filename to match (defaults to .*)
+ :param dry_run a boolean to enable dry run execution (defaults to False)
+ """
+ print(f"**** cleaning bucket {bucket} content matching {pattern}. Dry run mode {dry_run}")
+
+ mo_client = mutil.build_mo_client(self._minio_host, self._minio_port,self._minio_access_key, self._minio_secret_key)
+ result = []
+ objects = mo_client.list_objects(bucket_name=bucket, recursive= True)
+
+ for obj in objects:
+ if re.search(pattern, obj.object_name):
+ if "true" in dry_run:
+ result.append({"name":obj.object_name,"last_modified": str(obj.last_modified), "size":obj.size})
+ else:
+ removed = mutil.rm_file(mo_client,bucket,obj.object_name)
+ if removed:
+ result.append({"name":obj.object_name})
+ else:
+ print(f"skipping {obj.object_name} as it does not match {pattern}")
+
+ input.result(result)
+ input.status(200)
+
+ def execute(self, input:CommandData):
+ print(f"**** Minio command to execute {input.command()}")
+ try:
+ input.status(400)
+ input.result(f"could not execute minio command {input.command()} invalid arguments.")
+
+ if "ls" in input.command() and not "args" in input.get_metadata():
+ self._list_buckets(input)
+
+ if "ls" in input.command() and "args" in input.get_metadata() and len(input.args()) == 1:
+ self._list_bucket_content(input.args()[0],input)
+
+ if "rm" in input.command() and "args" in input.get_metadata() and len(input.args()) == 2:
+ self._rm_bucket_object(input.args()[0],input.args()[1],input)
+
+ if "mv" in input.command() and "args" in input.get_metadata() and len(input.args()) == 4:
+ self._mv_bucket_object(input.args()[0],input.args()[1],input.args()[2],input.args()[3], input)
+
+ if "cp" in input.command() and "args" in input.get_metadata() and len(input.args()) == 4:
+ self._cp_bucket_object(input.args()[0],input.args()[1],input.args()[2],input.args()[3], input)
+
+ if "clean" in input.command() and "args" in input.get_metadata() and len(input.args()) == 3:
+ self._clean_bucket_content(input,input.args()[0],input.args()[1],input.args()[2])
+
+ except Exception as e:
+ input.result(f"could not execute minio command {e}")
+ input.status(400)
+
+ return input
\ No newline at end of file
diff --git a/actions/devel/minio/common/authorize.py b/actions/devel/minio/common/authorize.py
new file mode 100644
index 0000000..f7d716f
--- /dev/null
+++ b/actions/devel/minio/common/authorize.py
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from base64 import b64decode, b64encode
+from urllib.parse import quote, unquote
+
+import nuvolaris.config as cfg
+import nuvolaris.couchdb_util as cu
+import json
+import common.util as ut
+
+USER_META_DBN = "users_metadata"
+SUBJECT_META_DBN = "subjects"
+
+class DecodeError(Exception):
+ pass
+
+class EncodeError(Exception):
+ pass
+
+class AuthorizationError(Exception):
+ pass
+
+class Authorize():
+
+ def __init__(self, cdb_host, cdb_user, cdb_pwd):
+ cfg.clean()
+ cfg.put("couchdb.host", cdb_host)
+ cfg.put("couchdb.admin.user", cdb_user)
+ cfg.put("couchdb.admin.password", cdb_pwd)
+ self._db = cu.CouchDB()
+
+ def encode(self,username, password):
+ """Returns an HTTP basic authentication encrypted string given a valid
+ username and password.
+ """
+ if ':' in username:
+ raise EncodeError
+
+ username_password = f'{quote(username)}:{quote(password)}'
+ return f'Basic {b64encode(username_password.encode()).decode()}'
+
+ def _parse_b64(self, encoded_str):
+ try:
+ username, password = b64decode(encoded_str).decode().split(':', 1)
+ except:
+ # fallback in case the token is not bas64 encoded
+ username, password = encoded_str.split(':', 1)
+
+ if not username or not password:
+ raise DecodeError("authentication token does not seems to be b64 encoded")
+
+ return username, password
+
+ def decode(self, encoded_str):
+ """Decode an encrypted HTTP basic authentication string. Returns a tuple of
+ the form (username, password), and raises a DecodeError exception if
+ nothing could be decoded.
+ """
+ split = encoded_str.strip().split(' ')
+
+ # If split is only one element, try to decode the username and password
+ # directly.
+ if len(split) == 1:
+ username, password = self._parse_b64(split[0])
+
+ # If there are only two elements, check the first and ensure it says
+ # 'basic' so that we know we're about to decode the right thing. If not,
+ # bail out.
+ elif len(split) == 2:
+ if split[0].strip().lower() == 'basic':
+ username, password = self._parse_b64(split[1])
+ else:
+ raise DecodeError("authentication token provides more than 2 elements. could not parse properly")
+
+ # If there are more than 2 elements, something crazy must be happening.
+ # Bail.
+ else:
+ raise DecodeError("unpredictable error parsing authentication token")
+
+ return unquote(username), unquote(password)
+
+ def fetch_subject(self, uuid: str, key: str):
+ """
+ Query the internal couchdb searching for the subject mathing the given uuid, key.
+ Normally these stored in wsk or wsku in the form uuid:key
+ :param uuid the OW subject uuid
+ :param key the OW subject key
+ :return a ubject document
+ """
+ print(f"searching for openwhisk subject {uuid}")
+ try:
+ selector= {
+ "selector": {"namespaces": {"$elemMatch": {"uuid": uuid,"key": key}}}
+ }
+
+ response = self._db.find_doc(SUBJECT_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris namespace for user {uuid} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {uuid} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {uuid}. Reason: {e}")
+ return None
+
+ def fetch_user_data(self, username: str):
+ """
+ Query the internal couchdb searching for the given principal to retrieve all the
+ relevant metadata
+ """
+ print(f"searching for user {username} meta-data")
+ try:
+ selector = {"selector":{"login": {"$eq": username }}}
+ response = self._db.find_doc(USER_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris metadata for user {username} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {username} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {username}. Reason: {e}")
+ return None
+
+ def login(self, authorization: str):
+ """
+ Attempt to login the user identified by the given Openwhisk authorization AUTH token as base64
+ """
+ uuid, key = self.decode(authorization)
+ subject = self.fetch_subject(uuid, key)
+
+ if not subject:
+ raise AuthorizationError("Openwhisk subject not found.")
+
+ user_data = self.fetch_user_data(subject['subject'])
+
+ if user_data:
+ return user_data
+
+ raise AuthorizationError("Could not retrieve user metadata.")
\ No newline at end of file
diff --git a/actions/devel/minio/common/command_data.py b/actions/devel/minio/common/command_data.py
new file mode 100644
index 0000000..44f3faa
--- /dev/null
+++ b/actions/devel/minio/common/command_data.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+import logging
+
+class CommandData:
+ _data = {}
+ _raw_data = {}
+
+ def __init__(self, cmd):
+ self._data = cmd
+ self._raw_data = cmd.copy()
+
+ def dump(self):
+ logging.debug(json.dumps(self._data))
+
+ def status(self, status):
+ self._data['status']=status
+
+ def result(self, result):
+ self._data['result']=result
+
+ def get_metadata(self):
+ return self._data
+
+ def get_raw_data(self):
+ return self._raw_data
+
+ def command(self):
+ return self._data['command']
+
+ def args(self):
+ return self._data['args']
+
+
diff --git a/actions/devel/minio/common/minio_util.py b/actions/devel/minio/common/minio_util.py
new file mode 100644
index 0000000..e1e8f90
--- /dev/null
+++ b/actions/devel/minio/common/minio_util.py
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import base64
+import random
+import string
+import mimetypes
+
+from minio import Minio
+from minio.error import S3Error
+from minio.commonconfig import CopySource
+
+def extract_mimetype(file):
+ mimetype, _ = mimetypes.guess_type(file)
+
+ if mimetype is None:
+ return "application/octet-stream"
+ else:
+ return mimetype
+
+def build_mo_client(host, port, access_key, secret_key):
+ """
+ Creates an Minio client pointing to the given MINIO HOST
+ :param host, minio host
+ :param port, minio api port normally it is the 9000
+ :param access_key user we are representing
+ :param secret_key to access minio
+ """
+ mo_client = Minio(f"{host}:{port}",access_key=access_key,secret_key=secret_key,secure=False)
+ return mo_client
+
+def upload_file(mo_client, file, bucket, object_name=None):
+ """Upload a file to an S3 bucket
+
+ :param file_name: File to upload
+ :param bucket: Bucket to upload to
+ :param object_name: S3 object name. If not specified then file_name is used
+ :return: a tuple with the Booleand str,
+ """
+
+ # If S3 object_name was not specified, use file_name
+ if object_name is None:
+ object_name = os.path.basename(file)
+
+ print(f"uploading {object_name} into bucket {bucket} from tmp_file {file}")
+
+ # Upload the file
+ try:
+ mimetype = extract_mimetype(object_name)
+ response = mo_client.fput_object(bucket_name=bucket,object_name=object_name,file_path=file,content_type=mimetype)
+ if response._object_name:
+ return True, response._object_name
+ except Exception as e:
+ print(e)
+ return False, str(e)
+ return False, "KO"
+
+def prepare_file_upload(username, filename, file_content_as_b64):
+ """ Creates a tmp area for the given user where the uploaded file will be stored under a random generated name
+ the fully qualified filename is taken into account only on the corresponding destination bucket.
+ param: username
+ param: filename
+ param: file_content_as_b64
+ return: a file object pointing to the tmp file
+ """
+ try:
+ user_tmp_folder = f"/tmp/{username}"
+ if not os.path.exists(user_tmp_folder):
+ os.makedirs(user_tmp_folder)
+ print(f"added tmp folder {user_tmp_folder}")
+
+ delete_files_in_directory_and_subdirectories(user_tmp_folder)
+ rnd_filename = get_random_string(20)
+ tmp_file = f"{user_tmp_folder}/{rnd_filename}"
+
+ with open(tmp_file, "wb") as f:
+ file_content=base64.b64decode(file_content_as_b64)
+ f.write(file_content)
+
+ if os.path.exists(tmp_file):
+ print(f"{tmp_file} stored successfully.")
+ return tmp_file
+ else:
+ return None
+ except Exception as e:
+ print("error preparing tmp_files",e)
+ return None
+
+def delete_files_in_directory_and_subdirectories(directory_path):
+ try:
+ for root, dirs, files in os.walk(directory_path):
+ for file in files:
+ file_path = os.path.join(root, file)
+ os.remove(file_path)
+ print("All files and subdirectories deleted successfully.")
+ except OSError:
+ print("Error occurred while deleting files and subdirectories.")
+
+def get_random_string(length):
+ # choose from all lowercase letter
+ letters = string.ascii_lowercase
+ return ''.join(random.choice(letters) for i in range(length))
+
+def rm_file(mo_client, bucket, file):
+ """ Remove a file from a bucket
+ :parma mo_client a minio client instance to execute the command
+ :param bucket the bucket name
+ :param file the file name
+ :return True if the file has been removed, False otherwise
+ """
+
+ try:
+ mo_client.remove_object(bucket, file)
+ return True
+ except Exception as e:
+ print(e)
+ return False
+ return False
+
+def cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Copy a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+ try:
+ # copy an object from a bucket to another.
+ cp_result = mo_client.copy_object(
+ dest_bucket,
+ dest_file,
+ CopySource(orig_bucket, orig_file)
+ )
+ if cp_result.object_name:
+ return cp_result.object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
+def mv_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Move a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+
+ # Upload the file
+ try:
+ # mv an object from a bucket to another.
+ object_name = cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file)
+ if object_name:
+ rm_file(mo_client,orig_bucket,orig_file)
+ return object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
diff --git a/actions/devel/minio/common/util.py b/actions/devel/minio/common/util.py
new file mode 100644
index 0000000..f1e8f39
--- /dev/null
+++ b/actions/devel/minio/common/util.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import mimetypes
+
+def get_env_value(user_data, key):
+ """
+ Check if inside the given user_data object there is an env item with the
+ given name and returns the value if any.
+ :param user_data
+ :param key
+ :return None if the given key it is not prese
+ """
+ envs = list(user_data['env'])
+
+ for env in envs:
+ if env['key'] == key:
+ return env['value']
+
+ return None
\ No newline at end of file
diff --git a/actions/devel/minio/flatdict.py b/actions/devel/minio/flatdict.py
new file mode 100644
index 0000000..834ce99
--- /dev/null
+++ b/actions/devel/minio/flatdict.py
@@ -0,0 +1,506 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""FlatDict is a dict object that allows for single level, delimited
+key/value pair mapping of nested dictionaries.
+
+"""
+try:
+ from collections.abc import MutableMapping
+except ImportError: # pragma: nocover
+ from collections import MutableMapping
+import sys
+
+__version__ = '4.0.1'
+
+NO_DEFAULT = object()
+
+
+class FlatDict(MutableMapping):
+ """:class:`~flatdict.FlatDict` is a dictionary object that allows for
+ single level, delimited key/value pair mapping of nested dictionaries.
+ The default delimiter value is ``:`` but can be changed in the constructor
+ or by calling :meth:`FlatDict.set_delimiter`.
+
+ """
+ _COERCE = dict
+
+ def __init__(self, value=None, delimiter=':', dict_class=dict):
+ super(FlatDict, self).__init__()
+ self._values = dict_class()
+ self._delimiter = delimiter
+ self.update(value)
+
+ def __contains__(self, key):
+ """Check to see if the key exists, checking for both delimited and
+ not delimited key values.
+
+ :param mixed key: The key to check for
+
+ """
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ return pk in self._values and ck in self._values[pk]
+ return key in self._values
+
+ def __delitem__(self, key):
+ """Delete the item for the specified key, automatically dealing with
+ nested children.
+
+ :param mixed key: The key to use
+ :raises: KeyError
+
+ """
+ if key not in self:
+ raise KeyError
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ del self._values[pk][ck]
+ if not self._values[pk]:
+ del self._values[pk]
+ else:
+ del self._values[key]
+
+ def __eq__(self, other):
+ """Check for equality against the other value
+
+ :param other: The value to compare
+ :type other: FlatDict
+ :rtype: bool
+ :raises: TypeError
+
+ """
+ if isinstance(other, dict):
+ return self.as_dict() == other
+ elif not isinstance(other, self.__class__):
+ raise TypeError
+ return self.as_dict() == other.as_dict()
+
+ def __ne__(self, other):
+ """Check for inequality against the other value
+
+ :param other: The value to compare
+ :type other: dict or FlatDict
+ :rtype: bool
+
+ """
+ return not self.__eq__(other)
+
+ def __getitem__(self, key):
+ """Get an item for the specified key, automatically dealing with
+ nested children.
+
+ :param mixed key: The key to use
+ :rtype: mixed
+ :raises: KeyError
+
+ """
+ values = self._values
+ key = [key] if isinstance(key, int) else key.split(self._delimiter)
+ for part in key:
+ values = values[part]
+ return values
+
+ def __iter__(self):
+ """Iterate over the flat dictionary key and values
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ return iter(self.keys())
+
+ def __len__(self):
+ """Return the number of items.
+
+ :rtype: int
+
+ """
+ return len(self.keys())
+
+ def __reduce__(self):
+ """Return state information for pickling
+
+ :rtype: tuple
+
+ """
+ return type(self), (self.as_dict(), self._delimiter)
+
+ def __repr__(self):
+ """Return the string representation of the instance.
+
+ :rtype: str
+
+ """
+ return '<{} id={} {}>"'.format(self.__class__.__name__, id(self),
+ str(self))
+
+ def __setitem__(self, key, value):
+ """Assign the value to the key, dynamically building nested
+ FlatDict items where appropriate.
+
+ :param mixed key: The key for the item
+ :param mixed value: The value for the item
+ :raises: TypeError
+
+ """
+ if isinstance(value, self._COERCE) and not isinstance(value, FlatDict):
+ value = self.__class__(value, self._delimiter)
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if pk not in self._values:
+ self._values[pk] = self.__class__({ck: value}, self._delimiter)
+ return
+ elif not isinstance(self._values[pk], FlatDict):
+ raise TypeError(
+ 'Assignment to invalid type for key {}'.format(pk))
+ self._values[pk][ck] = value
+ else:
+ self._values[key] = value
+
+ def __str__(self):
+ """Return the string value of the instance.
+
+ :rtype: str
+
+ """
+ return '{{{}}}'.format(', '.join(
+ ['{!r}: {!r}'.format(k, self[k]) for k in self.keys()]))
+
+ def as_dict(self):
+ """Return the :class:`~flatdict.FlatDict` as a :class:`dict`
+
+ :rtype: dict
+
+ """
+ out = dict({})
+ for key in self.keys():
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if self._has_delimiter(ck):
+ ck = ck.split(self._delimiter, 1)[0]
+ if isinstance(self._values[pk], FlatDict) and pk not in out:
+ out[pk] = {}
+ if isinstance(self._values[pk][ck], FlatDict):
+ out[pk][ck] = self._values[pk][ck].as_dict()
+ else:
+ out[pk][ck] = self._values[pk][ck]
+ else:
+ out[key] = self._values[key]
+ return out
+
+ def clear(self):
+ """Remove all items from the flat dictionary."""
+ self._values.clear()
+
+ def copy(self):
+ """Return a shallow copy of the flat dictionary.
+
+ :rtype: flatdict.FlatDict
+
+ """
+ return self.__class__(self.as_dict(), delimiter=self._delimiter)
+
+ def get(self, key, d=None):
+ """Return the value for key if key is in the flat dictionary, else
+ default. If default is not given, it defaults to ``None``, so that this
+ method never raises :exc:`KeyError`.
+
+ :param mixed key: The key to get
+ :param mixed d: The default value
+ :rtype: mixed
+
+ """
+ try:
+ return self.__getitem__(key)
+ except KeyError:
+ return d
+
+ def items(self):
+ """Return a copy of the flat dictionary's list of ``(key, value)``
+ pairs.
+
+ .. note:: CPython implementation detail: Keys and values are listed in
+ an arbitrary order which is non-random, varies across Python
+ implementations, and depends on the flat dictionary's history of
+ insertions and deletions.
+
+ :rtype: list
+
+ """
+ return [(k, self.__getitem__(k)) for k in self.keys()]
+
+ def iteritems(self):
+ """Return an iterator over the flat dictionary's (key, value) pairs.
+ See the note for :meth:`flatdict.FlatDict.items`.
+
+ Using ``iteritems()`` while adding or deleting entries in the flat
+ dictionary may raise :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for item in self.items():
+ yield item
+
+ def iterkeys(self):
+ """Iterate over the flat dictionary's keys. See the note for
+ :meth:`flatdict.FlatDict.items`.
+
+ Using ``iterkeys()`` while adding or deleting entries in the flat
+ dictionary may raise :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for key in self.keys():
+ yield key
+
+ def itervalues(self):
+ """Return an iterator over the flat dictionary's values. See the note
+ :meth:`flatdict.FlatDict.items`.
+
+ Using ``itervalues()`` while adding or deleting entries in the flat
+ dictionary may raise a :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for value in self.values():
+ yield value
+
+ def keys(self):
+ """Return a copy of the flat dictionary's list of keys.
+ See the note for :meth:`flatdict.FlatDict.items`.
+
+ :rtype: list
+
+ """
+ keys = []
+
+ for key, value in self._values.items():
+ if isinstance(value, (FlatDict, dict)):
+ nested = [
+ self._delimiter.join([str(key), str(k)])
+ for k in value.keys()]
+ keys += nested if nested else [key]
+ else:
+ keys.append(key)
+
+ return keys
+
+ def pop(self, key, default=NO_DEFAULT):
+ """If key is in the flat dictionary, remove it and return its value,
+ else return default. If default is not given and key is not in the
+ dictionary, :exc:`KeyError` is raised.
+
+ :param mixed key: The key name
+ :param mixed default: The default value
+ :rtype: mixed
+
+ """
+ if key not in self and default != NO_DEFAULT:
+ return default
+ value = self[key]
+ self.__delitem__(key)
+ return value
+
+ def setdefault(self, key, default):
+ """If key is in the flat dictionary, return its value. If not,
+ insert key with a value of default and return default.
+ default defaults to ``None``.
+
+ :param mixed key: The key name
+ :param mixed default: The default value
+ :rtype: mixed
+
+ """
+ if key not in self:
+ self.__setitem__(key, default)
+ return self.__getitem__(key)
+
+ def set_delimiter(self, delimiter):
+ """Override the default or passed in delimiter with a new value. If
+ the requested delimiter already exists in a key, a :exc:`ValueError`
+ will be raised.
+
+ :param str delimiter: The delimiter to use
+ :raises: ValueError
+
+ """
+ for key in self.keys():
+ if delimiter in key:
+ raise ValueError('Key {!r} collides with delimiter {!r}', key,
+ delimiter)
+ self._delimiter = delimiter
+ for key in self._values.keys():
+ if isinstance(self._values[key], FlatDict):
+ self._values[key].set_delimiter(delimiter)
+
+ def update(self, other=None, **kwargs):
+ """Update the flat dictionary with the key/value pairs from other,
+ overwriting existing keys.
+
+ ``update()`` accepts either another flat dictionary object or an
+ iterable of key/value pairs (as tuples or other iterables of length
+ two). If keyword arguments are specified, the flat dictionary is then
+ updated with those key/value pairs: ``d.update(red=1, blue=2)``.
+
+ :param iterable other: Iterable of key, value pairs
+ :rtype: None
+
+ """
+ [self.__setitem__(k, v) for k, v in dict(other or kwargs).items()]
+
+ def values(self):
+ """Return a copy of the flat dictionary's list of values. See the note
+ for :meth:`flatdict.FlatDict.items`.
+
+ :rtype: list
+
+ """
+ return [self.__getitem__(k) for k in self.keys()]
+
+ def _has_delimiter(self, key):
+ """Checks to see if the key contains the delimiter.
+
+ :rtype: bool
+
+ """
+ return isinstance(key, str) and self._delimiter in key
+
+
+class FlatterDict(FlatDict):
+ """Like :class:`~flatdict.FlatDict` but also coerces lists and sets
+ to child-dict instances with the offset as the key. Alternative to
+ the implementation added in v1.2 of FlatDict.
+
+ """
+ _COERCE = list, tuple, set, dict, FlatDict
+ _ARRAYS = list, set, tuple
+
+ def __init__(self, value=None, delimiter=':', dict_class=dict):
+ self.original_type = type(value)
+ if self.original_type in self._ARRAYS:
+ value = {str(i): v for i, v in enumerate(value)}
+ super(FlatterDict, self).__init__(value, delimiter, dict_class)
+
+ def __setitem__(self, key, value):
+ """Assign the value to the key, dynamically building nested
+ FlatDict items where appropriate.
+
+ :param mixed key: The key for the item
+ :param mixed value: The value for the item
+ :raises: TypeError
+
+ """
+ if isinstance(value, self._COERCE) and \
+ not isinstance(value, FlatterDict):
+ value = self.__class__(value, self._delimiter)
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if pk not in self._values:
+ self._values[pk] = self.__class__({ck: value}, self._delimiter)
+ return
+ if getattr(self._values[pk], 'original_type',
+ None) in self._ARRAYS:
+ try:
+ k, cck = ck.split(self._delimiter, 1)
+ int(k)
+ except ValueError:
+ raise TypeError(
+ 'Assignment to invalid type for key {}{}{}'.format(
+ pk, self._delimiter, ck))
+ self._values[pk][k][cck] = value
+ return
+ elif not isinstance(self._values[pk], FlatterDict):
+ raise TypeError(
+ 'Assignment to invalid type for key {}'.format(pk))
+ self._values[pk][ck] = value
+ else:
+ self._values[key] = value
+
+ def as_dict(self):
+ """Return the :class:`~flatdict.FlatterDict` as a nested
+ :class:`dict`.
+
+ :rtype: dict
+
+ """
+ out = {}
+ for key in self.keys():
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if self._has_delimiter(ck):
+ ck = ck.split(self._delimiter, 1)[0]
+ if isinstance(self._values[pk], FlatterDict) and pk not in out:
+ if self._values[pk].original_type == tuple:
+ out[pk] = tuple(self._child_as_list(pk))
+ elif self._values[pk].original_type == list:
+ out[pk] = self._child_as_list(pk)
+ elif self._values[pk].original_type == set:
+ out[pk] = set(self._child_as_list(pk))
+ elif self._values[pk].original_type == dict:
+ out[pk] = self._values[pk].as_dict()
+ else:
+ if isinstance(self._values[key], FlatterDict):
+ out[key] = self._values[key].original_type()
+ else:
+ out[key] = self._values[key]
+ return out
+
+ def _child_as_list(self, pk, ck=None):
+ """Returns a list of values from the child FlatterDict instance
+ with string based integer keys.
+
+ :param str pk: The parent key
+ :param str ck: The child key, optional
+ :rtype: list
+
+ """
+ if ck is None:
+ subset = self._values[pk]
+ else:
+ subset = self._values[pk][ck]
+ # Check if keys has delimiter, which implies deeply nested dict
+ keys = subset.keys()
+ if any(self._has_delimiter(k) for k in keys):
+ out = []
+ split_keys = {k.split(self._delimiter)[0] for k in keys}
+ for k in sorted(split_keys, key=lambda x: int(x)):
+ if subset[k].original_type == tuple:
+ out.append(tuple(self._child_as_list(pk, k)))
+ elif subset[k].original_type == list:
+ out.append(self._child_as_list(pk, k))
+ elif subset[k].original_type == set:
+ out.append(set(self._child_as_list(pk, k)))
+ elif subset[k].original_type == dict:
+ out.append(subset[k].as_dict())
+ return out
+
+ # Python prior 3.6 does not guarantee insertion order, remove it after
+ # EOL python 3.5 - 2020-09-13
+ if sys.version_info[0:2] < (3, 6): # pragma: nocover
+ return [subset[k] for k in sorted(keys, key=lambda x: int(x))]
+ else:
+ return [subset[k] for k in keys]
diff --git a/actions/devel/minio/nuvolaris/config.py b/actions/devel/minio/nuvolaris/config.py
new file mode 100644
index 0000000..787d4a7
--- /dev/null
+++ b/actions/devel/minio/nuvolaris/config.py
@@ -0,0 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import flatdict, json, os
+import logging
+
+_config = {}
+
+# define a configuration
+# the configuration is a map, followed by a list of labels
+# the map can be a serialized json and will be flattened to a map of values.
+# you can have only a configuration active at a time
+# if you want to set a new configuration you have to clean it
+def configure(spec: dict):
+ global _config
+ _config = dict(flatdict.FlatDict(spec, delimiter="."))
+
+ #forces autodetect of kube if not provided
+ if not exists('nuvolaris.kube'):
+ put('nuvolaris.kube','auto')
+
+ return True
+
+def clean():
+ global _config
+ _config = {}
+
+def exists(key):
+ return key in _config
+
+def get(key, envvar=None, defval=None):
+ val = _config.get(key)
+
+ if envvar and envvar in os.environ:
+ val = os.environ[envvar]
+
+ if val:
+ return val
+
+ return defval
+
+def put(key, value):
+ _config[key] = value
+ return True
+
+def delete(key):
+ if key in _config:
+ del _config[key]
+ return True
+
+def getall(prefix=""):
+ res = {}
+ for key in _config.keys():
+ if key.startswith(prefix):
+ res[key] = _config[key]
+ return res
+
+def keys(prefix=""):
+ res = []
+ if _config:
+ for key in _config.keys():
+ if key.startswith(prefix):
+ res.append(key)
+ return res
+
+def detect_labels(labels=None):
+ # skip autodetection of nuvolaris.kube if already available in _config
+ if exists('nuvolaris.kube') and get('nuvolaris.kube') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.kube={get('nuvolaris.kube')}")
+ return {}
+
+ # read labels if not available
+ if not labels:
+ import nuvolaris.kube as kube
+ labels = kube.kubectl("get", "nodes", jsonpath='{.items[].metadata.labels}')
+
+ res = {}
+ kube = None
+ for i in labels:
+ for j in list(i.keys()):
+ if j.startswith("nuvolaris.io/"):
+ key = f"nuvolaris.{j[13:]}"
+ res[key] = i[j]
+ _config[key] = i[j]
+
+ for i in labels:
+ for j in list(i.keys()):
+ # detect the kube type
+ if j.find("eksctl.io") >= 0:
+ kube ="eks"
+ break
+ if j.find("k8s.io/cloud-provider-aws") >= 0:
+ kube ="eks"
+ break
+ elif j.find("microk8s.io") >= 0:
+ kube = "microk8s"
+ break
+ elif j.find("lke.linode.com") >=0:
+ kube = "lks"
+ break
+ elif j.find("node.openshift.io") >=0:
+ kube = "openshift"
+ break
+ elif j.endswith("kubernetes.io/instance-type") and i[j] == "k3s":
+ kube = "k3s"
+ break
+ elif j.find("cloud.google.com/gke") >=0:
+ kube = "gke"
+ break
+ elif j.find("kubernetes.azure.com") >=0:
+ kube = "aks"
+ break
+ # assign all the 'nuvolaris.io' labels
+
+ if kube:
+ res["nuvolaris.kube"] = kube
+ _config["nuvolaris.kube"] = kube
+ return res
+
+ # defaults to generic if it is not yet detected
+ if exists('nuvolaris.kube') and get('nuvolaris.kube') == 'auto':
+ _config["nuvolaris.kube"] = "generic"
+ res["nuvolaris.kube"] = "generic"
+
+ return res
+
+def detect_storage(storages=None):
+ res = {}
+ if not storages:
+ import nuvolaris.util as util
+
+ try:
+ detect_storageclass = True
+ detect_provisioner = True
+
+ # skips autodetection if already provided
+ if exists('nuvolaris.storageclass') and get('nuvolaris.storageclass') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.storageclass={get('nuvolaris.storageclass')}")
+ detect_storageclass = False
+
+ if exists('nuvolaris.provisioner') and get('nuvolaris.provisioner') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.provisioner={get('nuvolaris.provisioner')}")
+ detect_provisioner = False
+
+ if not detect_storageclass and not detect_provisioner:
+ return res
+
+ if detect_storageclass:
+ storage_class = util.get_default_storage_class()
+ if storage_class:
+ res['nuvolaris.storageclass'] = storage_class
+ _config['nuvolaris.storageclass'] = storage_class
+
+ if detect_provisioner:
+ provisioner = util.get_default_storage_provisioner()
+ if(provisioner):
+ res['nuvolaris.provisioner'] = provisioner
+ _config['nuvolaris.provisioner'] = provisioner
+ except:
+ pass
+ return res
+
+def detect_env():
+ _config['operator.image'] = os.environ.get("OPERATOR_IMAGE", "missing-OPERATOR_IMAGE")
+ _config['operator.tag'] = os.environ.get("OPERATOR_TAG", "missing-OPERATOR_TAG")
+
+ # skip autodetection of controller.image if already configure into whisk.yaml
+ if not exists('controller.image'):
+ _config['controller.image'] = os.environ.get("CONTROLLER_IMAGE", "missing-CONTROLLER_IMAGE")
+ _config['controller.tag'] = os.environ.get("CONTROLLER_TAG", "missing-CONTROLLER_TAG")
+ else:
+ logging.warn(f"OW controller image detection skipped. Using {get('controller.image')}")
+
+ if not exists('invoker.image'):
+ _config['invoker.image'] = os.environ.get("INVOKER_IMAGE", "missing-INVOKER_IMAGE")
+ _config['invoker.tag'] = os.environ.get("INVOKER_TAG", "missing-INVOKER_TAG")
+ else:
+ logging.warn(f"OW invoker image detection skipped. Using {get('invoker.image')}")
+
+def detect():
+ detect_storage()
+ detect_labels()
+ detect_env()
+
+def dump_config():
+ import nuvolaris.config as cfg
+ for k in cfg.getall():
+ logging.debug(f"{k} = {cfg.get(k)}")
diff --git a/actions/devel/minio/nuvolaris/couchdb_util.py b/actions/devel/minio/nuvolaris/couchdb_util.py
new file mode 100644
index 0000000..3adb6b9
--- /dev/null
+++ b/actions/devel/minio/nuvolaris/couchdb_util.py
@@ -0,0 +1,164 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import os, json, time, sys, logging
+import requests as req
+import nuvolaris.config as cfg
+
+class CouchDB:
+ def __init__(self):
+ self.db_protocol = "http"
+ self.db_prefix = "nuvolaris_"
+ self.db_port = cfg.get("couchdb.port", "COUCHDB_SERVICE_PORT", "5984")
+ self.db_host = cfg.get("couchdb.host", "COUCHDB_SERVICE_HOST", "couchdb")
+ self.db_username = cfg.get("couchdb.admin.user", "COUCHDB_ADMIN_USER", "whisk_admin")
+ self.db_password = cfg.get("couchdb.admin.password", "COUCHDB_ADMIN_PASSWORD", "some_passw0rd")
+ self.db_auth = req.auth.HTTPBasicAuth(self.db_username,self.db_password)
+ self.db_url = f"{self.db_protocol}://{self.db_host}:{self.db_port}"
+ self.db_base = f"{self.db_url}/{self.db_prefix}"
+ self.db_session = req.Session()
+ self.db_session.auth = self.db_auth
+
+ def wait_db_ready(self, max_seconds):
+ logging.info("entering CouchDB.wait_db_ready()")
+ start = time.time()
+ delta = 0
+ session = req.Session()
+ while delta < max_seconds:
+ try:
+ r = session.get(f"{self.db_url}/_utils", timeout=5)
+ logging.info(f"CouchDB.wait_db_ready() got response code = {r.status_code}")
+ if r.status_code == 200:
+ return True
+ except Exception as e:
+ logging.info(f"waiting since: {delta} seconds")
+ delta = int(time.time() - start)
+ time.sleep(1)
+ return False
+
+ # check if database exists, return boolean
+ def check_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.head(url)
+ return r.status_code == 200
+
+ # delete database, return true if ok
+ def delete_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.delete(url)
+ return r.status_code == 200
+
+ # create db, return true if ok
+ def create_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.put(url)
+ return r.status_code == 201
+
+ # database="subjects"
+ def recreate_db(self, database, recreate=False):
+ msg = "recreate_db:"
+ exists = self.check_db(database)
+ if recreate and exists:
+ msg += " deleted"
+ self.delete_db(database)
+ if recreate or not exists:
+ msg += " created"
+ self.create_db(database)
+ return msg
+
+ def get_doc(self, database, id, user=None, password="", no_auth=False):
+ url = f"{self.db_base}{database}/{id}"
+ session = req.Session()
+ if no_auth:
+ db_auth=None
+ elif user:
+ db_auth=req.auth.HTTPBasicAuth(user, password)
+ else:
+ db_auth = self.db_auth
+ session.auth = db_auth
+ r = session.get(url)
+ if r.status_code == 200:
+ return json.loads(r.text)
+ return None
+
+ def update_doc(self, database, doc):
+ if '_id' in doc:
+ url = f"{self.db_base}{database}/{doc['_id']}"
+ cur = self.get_doc(database, doc['_id'])
+ if cur and '_rev' in cur:
+ doc['_rev'] = cur['_rev']
+ r = self.db_session.put(url, json=doc)
+ else:
+ r = self.db_session.put(url, json=doc)
+ return r.status_code in [200,201]
+ return False
+
+ def delete_doc(self, database, id):
+ cur = self.get_doc(database, id)
+ if cur and '_rev' in cur:
+ url = f"{self.db_base}{database}/{cur['_id']}?rev={cur['_rev']}"
+ r = self.db_session.delete(url)
+ return r.status_code == 200
+ return False
+
+ def configure_single_node(self):
+ url = f"{self.db_url}/_cluster_setup"
+ data = {"action": "enable_single_node", "singlenode": True, "bind_address": "0.0.0.0", "port": 5984}
+ r = self.db_session.post(url, json=data)
+ return r.status_code == 201
+
+ def configure_no_reduce_limit(self):
+ url = f"{self.db_url}/_node/_local/_config/query_server_config/reduce_limit"
+ data=b'"false"'
+ r = self.db_session.put(url, data=data)
+ return r.status_code == 200
+
+ def add_user(self, username: str, password: str):
+ userpass = {"name": username, "password": password, "roles": [], "type": "user"}
+ url = f"{self.db_url}/_users/org.couchdb.user:{username}"
+ res = self.db_session.put(url, json=userpass)
+ return res.status_code in [200, 201, 421]
+
+ #def add_role(self, database: str, members: list[str] = [], admins: list[str] =[]):
+ def add_role(self, database: str, members = [], admins =[]):
+ roles = {"admins": { "names": admins, "roles": [] }, "members": { "names": members, "roles": [] } }
+ url = f"{self.db_base}{database}/_security"
+ res = self.db_session.put(url, json=roles)
+ return res.status_code in [200, 201, 421]
+
+#
+# Submit a POST request to the _find endpoint using the specified selector
+#
+ def find_doc(self, database, selector, user=None, password="", no_auth=False):
+ url = f"{self.db_base}{database}/_find"
+ headers = {'Content-Type': 'application/json'}
+ session = req.Session()
+
+ if no_auth:
+ db_auth=None
+ elif user:
+ db_auth=req.auth.HTTPBasicAuth(user, password)
+ else:
+ db_auth = self.db_auth
+
+ session.auth=db_auth
+ r = session.post(url, headers=headers, data=selector)
+ if r.status_code == 200:
+ return json.loads(r.text)
+
+ logging.warn(f"query to {url} failed with {r.status_code}. Body {r.text}")
+ return None
\ No newline at end of file
diff --git a/actions/devel/psql/__main__.py b/actions/devel/psql/__main__.py
new file mode 100644
index 0000000..2ba865e
--- /dev/null
+++ b/actions/devel/psql/__main__.py
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import nuvolaris.config as cfg
+import nuvolaris.couchdb_util as cu
+import logging, json
+
+from common.authorize import Authorize
+from common.command_data import CommandData
+from command.psql import Psql
+from base64 import b64decode
+
+class ApiError(Exception):
+ pass
+
+def build_error(message: str):
+ return {
+ "statusCode": 400,
+ "body": message
+ }
+
+def build_response(data:CommandData):
+ meta_data = data.get_metadata()
+ return {
+ "statusCode": meta_data['status'],
+ "body": meta_data['result']
+ }
+
+def parse_body(args):
+ try:
+ return b64decode(args['__ow_body']).decode().strip()
+ except Exception as e:
+ print(e)
+ raise ApiError("could not parse __ow_body as base64")
+
+def main(args):
+ """
+ Action implementing a generic command wrapper for the nuv devel plugin. The action must be called with a POST request receiving a JSON
+ payload similar to this one
+ {
+ "command":"select * from table"
+ }
+
+ the invoker must provide a x-impersonate-auth header containing the Openwhisk BASIC authentication of the wsku/user the action should impersonate
+ when submitting the command. Every specific provider will support specific request type and command.
+ WARNING: the body will be received as base64 encoded string as this action will be deployed with --web raw enabled flag
+ """
+ headers = args['__ow_headers']
+ if('x-impersonate-auth' not in headers):
+ return build_error("invalid request, missing mandatory header: x-impersonate-auth")
+
+ if(len(args['__ow_body']) == 0):
+ return build_error("invalid request, no command payload received")
+
+ try:
+ user_data = Authorize(args['couchdb_host'],args['couchdb_user'],args['couchdb_password']).login(headers['x-impersonate-auth'])
+ cmd = CommandData(json.loads(parse_body(args)))
+ return build_response(Psql(user_data).execute(cmd))
+ except Exception as e:
+ return build_error(f"failed to execute nuv devel psql. Reason: {str(e)}")
\ No newline at end of file
diff --git a/actions/devel/psql/command/psql.py b/actions/devel/psql/command/psql.py
new file mode 100644
index 0000000..8067802
--- /dev/null
+++ b/actions/devel/psql/command/psql.py
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import psycopg
+import common.util as ut
+
+from common.command_data import CommandData
+from psycopg.rows import dict_row
+
+class Psql():
+ """
+ Implementation of a Postgres Command executor. It will require
+ a user_data dictionary linked to a specific user.
+ """
+
+ def __init__(self, user_data):
+ self._user_data = user_data
+ self._postgres_url= ut.get_env_value(user_data,"POSTGRES_URL")
+ self.validate()
+
+ def validate(self):
+ """
+ Validate that the provided user_data contains the appropriate
+ metadata for being able to submit a postgres command.
+ """
+ if not self._postgres_url:
+ raise Exception("user does not have valid POSTGRES environment set")
+
+
+ def _query(self, input:CommandData):
+ """
+ Queries for matching query records and returns datas in a key, value format.
+ """
+ query = input.command()
+ with psycopg.connect(self._postgres_url) as conn:
+ # Open a cursor to perform database operations
+ with conn.cursor(row_factory=dict_row) as cur:
+ cur.execute(query)
+ result = cur.fetchall()
+ input.result(str(result))
+ input.status(200)
+ return input
+
+ def _script(self, input:CommandData):
+ script = input.command()
+ with psycopg.connect(self._postgres_url) as conn:
+ # Open a cursor to perform database operations
+ with conn.cursor() as cur:
+ cur.execute(script)
+ conn.commit()
+ input.result(cur.statusmessage)
+ input.status(200)
+ return input
+
+ def _is_a_query(self, input:CommandData):
+ return 'select' in input.command().lower()
+
+ def execute(self, input:CommandData):
+ print(f"**** Psql command to execute {input.command()}")
+ try:
+ if self._is_a_query(input):
+ return self._query(input)
+ else:
+ return self._script(input)
+ except Exception as e:
+ input.result(f"could not execute psql command {e}")
+ input.status(400)
+
+ return input
\ No newline at end of file
diff --git a/actions/devel/psql/common/authorize.py b/actions/devel/psql/common/authorize.py
new file mode 100644
index 0000000..f7d716f
--- /dev/null
+++ b/actions/devel/psql/common/authorize.py
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from base64 import b64decode, b64encode
+from urllib.parse import quote, unquote
+
+import nuvolaris.config as cfg
+import nuvolaris.couchdb_util as cu
+import json
+import common.util as ut
+
+USER_META_DBN = "users_metadata"
+SUBJECT_META_DBN = "subjects"
+
+class DecodeError(Exception):
+ pass
+
+class EncodeError(Exception):
+ pass
+
+class AuthorizationError(Exception):
+ pass
+
+class Authorize():
+
+ def __init__(self, cdb_host, cdb_user, cdb_pwd):
+ cfg.clean()
+ cfg.put("couchdb.host", cdb_host)
+ cfg.put("couchdb.admin.user", cdb_user)
+ cfg.put("couchdb.admin.password", cdb_pwd)
+ self._db = cu.CouchDB()
+
+ def encode(self,username, password):
+ """Returns an HTTP basic authentication encrypted string given a valid
+ username and password.
+ """
+ if ':' in username:
+ raise EncodeError
+
+ username_password = f'{quote(username)}:{quote(password)}'
+ return f'Basic {b64encode(username_password.encode()).decode()}'
+
+ def _parse_b64(self, encoded_str):
+ try:
+ username, password = b64decode(encoded_str).decode().split(':', 1)
+ except:
+ # fallback in case the token is not bas64 encoded
+ username, password = encoded_str.split(':', 1)
+
+ if not username or not password:
+ raise DecodeError("authentication token does not seems to be b64 encoded")
+
+ return username, password
+
+ def decode(self, encoded_str):
+ """Decode an encrypted HTTP basic authentication string. Returns a tuple of
+ the form (username, password), and raises a DecodeError exception if
+ nothing could be decoded.
+ """
+ split = encoded_str.strip().split(' ')
+
+ # If split is only one element, try to decode the username and password
+ # directly.
+ if len(split) == 1:
+ username, password = self._parse_b64(split[0])
+
+ # If there are only two elements, check the first and ensure it says
+ # 'basic' so that we know we're about to decode the right thing. If not,
+ # bail out.
+ elif len(split) == 2:
+ if split[0].strip().lower() == 'basic':
+ username, password = self._parse_b64(split[1])
+ else:
+ raise DecodeError("authentication token provides more than 2 elements. could not parse properly")
+
+ # If there are more than 2 elements, something crazy must be happening.
+ # Bail.
+ else:
+ raise DecodeError("unpredictable error parsing authentication token")
+
+ return unquote(username), unquote(password)
+
+ def fetch_subject(self, uuid: str, key: str):
+ """
+ Query the internal couchdb searching for the subject mathing the given uuid, key.
+ Normally these stored in wsk or wsku in the form uuid:key
+ :param uuid the OW subject uuid
+ :param key the OW subject key
+ :return a ubject document
+ """
+ print(f"searching for openwhisk subject {uuid}")
+ try:
+ selector= {
+ "selector": {"namespaces": {"$elemMatch": {"uuid": uuid,"key": key}}}
+ }
+
+ response = self._db.find_doc(SUBJECT_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris namespace for user {uuid} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {uuid} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {uuid}. Reason: {e}")
+ return None
+
+ def fetch_user_data(self, username: str):
+ """
+ Query the internal couchdb searching for the given principal to retrieve all the
+ relevant metadata
+ """
+ print(f"searching for user {username} meta-data")
+ try:
+ selector = {"selector":{"login": {"$eq": username }}}
+ response = self._db.find_doc(USER_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris metadata for user {username} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {username} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {username}. Reason: {e}")
+ return None
+
+ def login(self, authorization: str):
+ """
+ Attempt to login the user identified by the given Openwhisk authorization AUTH token as base64
+ """
+ uuid, key = self.decode(authorization)
+ subject = self.fetch_subject(uuid, key)
+
+ if not subject:
+ raise AuthorizationError("Openwhisk subject not found.")
+
+ user_data = self.fetch_user_data(subject['subject'])
+
+ if user_data:
+ return user_data
+
+ raise AuthorizationError("Could not retrieve user metadata.")
\ No newline at end of file
diff --git a/actions/devel/psql/common/command_data.py b/actions/devel/psql/common/command_data.py
new file mode 100644
index 0000000..44f3faa
--- /dev/null
+++ b/actions/devel/psql/common/command_data.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+import logging
+
+class CommandData:
+ _data = {}
+ _raw_data = {}
+
+ def __init__(self, cmd):
+ self._data = cmd
+ self._raw_data = cmd.copy()
+
+ def dump(self):
+ logging.debug(json.dumps(self._data))
+
+ def status(self, status):
+ self._data['status']=status
+
+ def result(self, result):
+ self._data['result']=result
+
+ def get_metadata(self):
+ return self._data
+
+ def get_raw_data(self):
+ return self._raw_data
+
+ def command(self):
+ return self._data['command']
+
+ def args(self):
+ return self._data['args']
+
+
diff --git a/actions/devel/psql/common/minio_util.py b/actions/devel/psql/common/minio_util.py
new file mode 100644
index 0000000..e1e8f90
--- /dev/null
+++ b/actions/devel/psql/common/minio_util.py
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import base64
+import random
+import string
+import mimetypes
+
+from minio import Minio
+from minio.error import S3Error
+from minio.commonconfig import CopySource
+
+def extract_mimetype(file):
+ mimetype, _ = mimetypes.guess_type(file)
+
+ if mimetype is None:
+ return "application/octet-stream"
+ else:
+ return mimetype
+
+def build_mo_client(host, port, access_key, secret_key):
+ """
+ Creates an Minio client pointing to the given MINIO HOST
+ :param host, minio host
+ :param port, minio api port normally it is the 9000
+ :param access_key user we are representing
+ :param secret_key to access minio
+ """
+ mo_client = Minio(f"{host}:{port}",access_key=access_key,secret_key=secret_key,secure=False)
+ return mo_client
+
+def upload_file(mo_client, file, bucket, object_name=None):
+ """Upload a file to an S3 bucket
+
+ :param file_name: File to upload
+ :param bucket: Bucket to upload to
+ :param object_name: S3 object name. If not specified then file_name is used
+ :return: a tuple with the Booleand str,
+ """
+
+ # If S3 object_name was not specified, use file_name
+ if object_name is None:
+ object_name = os.path.basename(file)
+
+ print(f"uploading {object_name} into bucket {bucket} from tmp_file {file}")
+
+ # Upload the file
+ try:
+ mimetype = extract_mimetype(object_name)
+ response = mo_client.fput_object(bucket_name=bucket,object_name=object_name,file_path=file,content_type=mimetype)
+ if response._object_name:
+ return True, response._object_name
+ except Exception as e:
+ print(e)
+ return False, str(e)
+ return False, "KO"
+
+def prepare_file_upload(username, filename, file_content_as_b64):
+ """ Creates a tmp area for the given user where the uploaded file will be stored under a random generated name
+ the fully qualified filename is taken into account only on the corresponding destination bucket.
+ param: username
+ param: filename
+ param: file_content_as_b64
+ return: a file object pointing to the tmp file
+ """
+ try:
+ user_tmp_folder = f"/tmp/{username}"
+ if not os.path.exists(user_tmp_folder):
+ os.makedirs(user_tmp_folder)
+ print(f"added tmp folder {user_tmp_folder}")
+
+ delete_files_in_directory_and_subdirectories(user_tmp_folder)
+ rnd_filename = get_random_string(20)
+ tmp_file = f"{user_tmp_folder}/{rnd_filename}"
+
+ with open(tmp_file, "wb") as f:
+ file_content=base64.b64decode(file_content_as_b64)
+ f.write(file_content)
+
+ if os.path.exists(tmp_file):
+ print(f"{tmp_file} stored successfully.")
+ return tmp_file
+ else:
+ return None
+ except Exception as e:
+ print("error preparing tmp_files",e)
+ return None
+
+def delete_files_in_directory_and_subdirectories(directory_path):
+ try:
+ for root, dirs, files in os.walk(directory_path):
+ for file in files:
+ file_path = os.path.join(root, file)
+ os.remove(file_path)
+ print("All files and subdirectories deleted successfully.")
+ except OSError:
+ print("Error occurred while deleting files and subdirectories.")
+
+def get_random_string(length):
+ # choose from all lowercase letter
+ letters = string.ascii_lowercase
+ return ''.join(random.choice(letters) for i in range(length))
+
+def rm_file(mo_client, bucket, file):
+ """ Remove a file from a bucket
+ :parma mo_client a minio client instance to execute the command
+ :param bucket the bucket name
+ :param file the file name
+ :return True if the file has been removed, False otherwise
+ """
+
+ try:
+ mo_client.remove_object(bucket, file)
+ return True
+ except Exception as e:
+ print(e)
+ return False
+ return False
+
+def cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Copy a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+ try:
+ # copy an object from a bucket to another.
+ cp_result = mo_client.copy_object(
+ dest_bucket,
+ dest_file,
+ CopySource(orig_bucket, orig_file)
+ )
+ if cp_result.object_name:
+ return cp_result.object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
+def mv_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file):
+ """Move a file between two buckets
+
+ :param orig_bucket: Origin bucket
+ :param orig_file: Origin file
+ :param dest_bucket: Destination bucket
+ :param dest_file: Destination file
+ :return: name of the destination file is everything is ok. None otherwise
+ """
+ print(f"**** moving file {orig_file} from bucket {orig_bucket} to bucket {dest_bucket} with name {dest_file}")
+
+ # Upload the file
+ try:
+ # mv an object from a bucket to another.
+ object_name = cp_file(mo_client, orig_bucket, orig_file, dest_bucket, dest_file)
+ if object_name:
+ rm_file(mo_client,orig_bucket,orig_file)
+ return object_name
+ except Exception as e:
+ print(e)
+ return None
+ return None
+
diff --git a/actions/devel/psql/common/util.py b/actions/devel/psql/common/util.py
new file mode 100644
index 0000000..f1e8f39
--- /dev/null
+++ b/actions/devel/psql/common/util.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import mimetypes
+
+def get_env_value(user_data, key):
+ """
+ Check if inside the given user_data object there is an env item with the
+ given name and returns the value if any.
+ :param user_data
+ :param key
+ :return None if the given key it is not prese
+ """
+ envs = list(user_data['env'])
+
+ for env in envs:
+ if env['key'] == key:
+ return env['value']
+
+ return None
\ No newline at end of file
diff --git a/actions/devel/psql/flatdict.py b/actions/devel/psql/flatdict.py
new file mode 100644
index 0000000..834ce99
--- /dev/null
+++ b/actions/devel/psql/flatdict.py
@@ -0,0 +1,506 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""FlatDict is a dict object that allows for single level, delimited
+key/value pair mapping of nested dictionaries.
+
+"""
+try:
+ from collections.abc import MutableMapping
+except ImportError: # pragma: nocover
+ from collections import MutableMapping
+import sys
+
+__version__ = '4.0.1'
+
+NO_DEFAULT = object()
+
+
+class FlatDict(MutableMapping):
+ """:class:`~flatdict.FlatDict` is a dictionary object that allows for
+ single level, delimited key/value pair mapping of nested dictionaries.
+ The default delimiter value is ``:`` but can be changed in the constructor
+ or by calling :meth:`FlatDict.set_delimiter`.
+
+ """
+ _COERCE = dict
+
+ def __init__(self, value=None, delimiter=':', dict_class=dict):
+ super(FlatDict, self).__init__()
+ self._values = dict_class()
+ self._delimiter = delimiter
+ self.update(value)
+
+ def __contains__(self, key):
+ """Check to see if the key exists, checking for both delimited and
+ not delimited key values.
+
+ :param mixed key: The key to check for
+
+ """
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ return pk in self._values and ck in self._values[pk]
+ return key in self._values
+
+ def __delitem__(self, key):
+ """Delete the item for the specified key, automatically dealing with
+ nested children.
+
+ :param mixed key: The key to use
+ :raises: KeyError
+
+ """
+ if key not in self:
+ raise KeyError
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ del self._values[pk][ck]
+ if not self._values[pk]:
+ del self._values[pk]
+ else:
+ del self._values[key]
+
+ def __eq__(self, other):
+ """Check for equality against the other value
+
+ :param other: The value to compare
+ :type other: FlatDict
+ :rtype: bool
+ :raises: TypeError
+
+ """
+ if isinstance(other, dict):
+ return self.as_dict() == other
+ elif not isinstance(other, self.__class__):
+ raise TypeError
+ return self.as_dict() == other.as_dict()
+
+ def __ne__(self, other):
+ """Check for inequality against the other value
+
+ :param other: The value to compare
+ :type other: dict or FlatDict
+ :rtype: bool
+
+ """
+ return not self.__eq__(other)
+
+ def __getitem__(self, key):
+ """Get an item for the specified key, automatically dealing with
+ nested children.
+
+ :param mixed key: The key to use
+ :rtype: mixed
+ :raises: KeyError
+
+ """
+ values = self._values
+ key = [key] if isinstance(key, int) else key.split(self._delimiter)
+ for part in key:
+ values = values[part]
+ return values
+
+ def __iter__(self):
+ """Iterate over the flat dictionary key and values
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ return iter(self.keys())
+
+ def __len__(self):
+ """Return the number of items.
+
+ :rtype: int
+
+ """
+ return len(self.keys())
+
+ def __reduce__(self):
+ """Return state information for pickling
+
+ :rtype: tuple
+
+ """
+ return type(self), (self.as_dict(), self._delimiter)
+
+ def __repr__(self):
+ """Return the string representation of the instance.
+
+ :rtype: str
+
+ """
+ return '<{} id={} {}>"'.format(self.__class__.__name__, id(self),
+ str(self))
+
+ def __setitem__(self, key, value):
+ """Assign the value to the key, dynamically building nested
+ FlatDict items where appropriate.
+
+ :param mixed key: The key for the item
+ :param mixed value: The value for the item
+ :raises: TypeError
+
+ """
+ if isinstance(value, self._COERCE) and not isinstance(value, FlatDict):
+ value = self.__class__(value, self._delimiter)
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if pk not in self._values:
+ self._values[pk] = self.__class__({ck: value}, self._delimiter)
+ return
+ elif not isinstance(self._values[pk], FlatDict):
+ raise TypeError(
+ 'Assignment to invalid type for key {}'.format(pk))
+ self._values[pk][ck] = value
+ else:
+ self._values[key] = value
+
+ def __str__(self):
+ """Return the string value of the instance.
+
+ :rtype: str
+
+ """
+ return '{{{}}}'.format(', '.join(
+ ['{!r}: {!r}'.format(k, self[k]) for k in self.keys()]))
+
+ def as_dict(self):
+ """Return the :class:`~flatdict.FlatDict` as a :class:`dict`
+
+ :rtype: dict
+
+ """
+ out = dict({})
+ for key in self.keys():
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if self._has_delimiter(ck):
+ ck = ck.split(self._delimiter, 1)[0]
+ if isinstance(self._values[pk], FlatDict) and pk not in out:
+ out[pk] = {}
+ if isinstance(self._values[pk][ck], FlatDict):
+ out[pk][ck] = self._values[pk][ck].as_dict()
+ else:
+ out[pk][ck] = self._values[pk][ck]
+ else:
+ out[key] = self._values[key]
+ return out
+
+ def clear(self):
+ """Remove all items from the flat dictionary."""
+ self._values.clear()
+
+ def copy(self):
+ """Return a shallow copy of the flat dictionary.
+
+ :rtype: flatdict.FlatDict
+
+ """
+ return self.__class__(self.as_dict(), delimiter=self._delimiter)
+
+ def get(self, key, d=None):
+ """Return the value for key if key is in the flat dictionary, else
+ default. If default is not given, it defaults to ``None``, so that this
+ method never raises :exc:`KeyError`.
+
+ :param mixed key: The key to get
+ :param mixed d: The default value
+ :rtype: mixed
+
+ """
+ try:
+ return self.__getitem__(key)
+ except KeyError:
+ return d
+
+ def items(self):
+ """Return a copy of the flat dictionary's list of ``(key, value)``
+ pairs.
+
+ .. note:: CPython implementation detail: Keys and values are listed in
+ an arbitrary order which is non-random, varies across Python
+ implementations, and depends on the flat dictionary's history of
+ insertions and deletions.
+
+ :rtype: list
+
+ """
+ return [(k, self.__getitem__(k)) for k in self.keys()]
+
+ def iteritems(self):
+ """Return an iterator over the flat dictionary's (key, value) pairs.
+ See the note for :meth:`flatdict.FlatDict.items`.
+
+ Using ``iteritems()`` while adding or deleting entries in the flat
+ dictionary may raise :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for item in self.items():
+ yield item
+
+ def iterkeys(self):
+ """Iterate over the flat dictionary's keys. See the note for
+ :meth:`flatdict.FlatDict.items`.
+
+ Using ``iterkeys()`` while adding or deleting entries in the flat
+ dictionary may raise :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for key in self.keys():
+ yield key
+
+ def itervalues(self):
+ """Return an iterator over the flat dictionary's values. See the note
+ :meth:`flatdict.FlatDict.items`.
+
+ Using ``itervalues()`` while adding or deleting entries in the flat
+ dictionary may raise a :exc:`RuntimeError` or fail to iterate over all
+ entries.
+
+ :rtype: Iterator
+ :raises: RuntimeError
+
+ """
+ for value in self.values():
+ yield value
+
+ def keys(self):
+ """Return a copy of the flat dictionary's list of keys.
+ See the note for :meth:`flatdict.FlatDict.items`.
+
+ :rtype: list
+
+ """
+ keys = []
+
+ for key, value in self._values.items():
+ if isinstance(value, (FlatDict, dict)):
+ nested = [
+ self._delimiter.join([str(key), str(k)])
+ for k in value.keys()]
+ keys += nested if nested else [key]
+ else:
+ keys.append(key)
+
+ return keys
+
+ def pop(self, key, default=NO_DEFAULT):
+ """If key is in the flat dictionary, remove it and return its value,
+ else return default. If default is not given and key is not in the
+ dictionary, :exc:`KeyError` is raised.
+
+ :param mixed key: The key name
+ :param mixed default: The default value
+ :rtype: mixed
+
+ """
+ if key not in self and default != NO_DEFAULT:
+ return default
+ value = self[key]
+ self.__delitem__(key)
+ return value
+
+ def setdefault(self, key, default):
+ """If key is in the flat dictionary, return its value. If not,
+ insert key with a value of default and return default.
+ default defaults to ``None``.
+
+ :param mixed key: The key name
+ :param mixed default: The default value
+ :rtype: mixed
+
+ """
+ if key not in self:
+ self.__setitem__(key, default)
+ return self.__getitem__(key)
+
+ def set_delimiter(self, delimiter):
+ """Override the default or passed in delimiter with a new value. If
+ the requested delimiter already exists in a key, a :exc:`ValueError`
+ will be raised.
+
+ :param str delimiter: The delimiter to use
+ :raises: ValueError
+
+ """
+ for key in self.keys():
+ if delimiter in key:
+ raise ValueError('Key {!r} collides with delimiter {!r}', key,
+ delimiter)
+ self._delimiter = delimiter
+ for key in self._values.keys():
+ if isinstance(self._values[key], FlatDict):
+ self._values[key].set_delimiter(delimiter)
+
+ def update(self, other=None, **kwargs):
+ """Update the flat dictionary with the key/value pairs from other,
+ overwriting existing keys.
+
+ ``update()`` accepts either another flat dictionary object or an
+ iterable of key/value pairs (as tuples or other iterables of length
+ two). If keyword arguments are specified, the flat dictionary is then
+ updated with those key/value pairs: ``d.update(red=1, blue=2)``.
+
+ :param iterable other: Iterable of key, value pairs
+ :rtype: None
+
+ """
+ [self.__setitem__(k, v) for k, v in dict(other or kwargs).items()]
+
+ def values(self):
+ """Return a copy of the flat dictionary's list of values. See the note
+ for :meth:`flatdict.FlatDict.items`.
+
+ :rtype: list
+
+ """
+ return [self.__getitem__(k) for k in self.keys()]
+
+ def _has_delimiter(self, key):
+ """Checks to see if the key contains the delimiter.
+
+ :rtype: bool
+
+ """
+ return isinstance(key, str) and self._delimiter in key
+
+
+class FlatterDict(FlatDict):
+ """Like :class:`~flatdict.FlatDict` but also coerces lists and sets
+ to child-dict instances with the offset as the key. Alternative to
+ the implementation added in v1.2 of FlatDict.
+
+ """
+ _COERCE = list, tuple, set, dict, FlatDict
+ _ARRAYS = list, set, tuple
+
+ def __init__(self, value=None, delimiter=':', dict_class=dict):
+ self.original_type = type(value)
+ if self.original_type in self._ARRAYS:
+ value = {str(i): v for i, v in enumerate(value)}
+ super(FlatterDict, self).__init__(value, delimiter, dict_class)
+
+ def __setitem__(self, key, value):
+ """Assign the value to the key, dynamically building nested
+ FlatDict items where appropriate.
+
+ :param mixed key: The key for the item
+ :param mixed value: The value for the item
+ :raises: TypeError
+
+ """
+ if isinstance(value, self._COERCE) and \
+ not isinstance(value, FlatterDict):
+ value = self.__class__(value, self._delimiter)
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if pk not in self._values:
+ self._values[pk] = self.__class__({ck: value}, self._delimiter)
+ return
+ if getattr(self._values[pk], 'original_type',
+ None) in self._ARRAYS:
+ try:
+ k, cck = ck.split(self._delimiter, 1)
+ int(k)
+ except ValueError:
+ raise TypeError(
+ 'Assignment to invalid type for key {}{}{}'.format(
+ pk, self._delimiter, ck))
+ self._values[pk][k][cck] = value
+ return
+ elif not isinstance(self._values[pk], FlatterDict):
+ raise TypeError(
+ 'Assignment to invalid type for key {}'.format(pk))
+ self._values[pk][ck] = value
+ else:
+ self._values[key] = value
+
+ def as_dict(self):
+ """Return the :class:`~flatdict.FlatterDict` as a nested
+ :class:`dict`.
+
+ :rtype: dict
+
+ """
+ out = {}
+ for key in self.keys():
+ if self._has_delimiter(key):
+ pk, ck = key.split(self._delimiter, 1)
+ if self._has_delimiter(ck):
+ ck = ck.split(self._delimiter, 1)[0]
+ if isinstance(self._values[pk], FlatterDict) and pk not in out:
+ if self._values[pk].original_type == tuple:
+ out[pk] = tuple(self._child_as_list(pk))
+ elif self._values[pk].original_type == list:
+ out[pk] = self._child_as_list(pk)
+ elif self._values[pk].original_type == set:
+ out[pk] = set(self._child_as_list(pk))
+ elif self._values[pk].original_type == dict:
+ out[pk] = self._values[pk].as_dict()
+ else:
+ if isinstance(self._values[key], FlatterDict):
+ out[key] = self._values[key].original_type()
+ else:
+ out[key] = self._values[key]
+ return out
+
+ def _child_as_list(self, pk, ck=None):
+ """Returns a list of values from the child FlatterDict instance
+ with string based integer keys.
+
+ :param str pk: The parent key
+ :param str ck: The child key, optional
+ :rtype: list
+
+ """
+ if ck is None:
+ subset = self._values[pk]
+ else:
+ subset = self._values[pk][ck]
+ # Check if keys has delimiter, which implies deeply nested dict
+ keys = subset.keys()
+ if any(self._has_delimiter(k) for k in keys):
+ out = []
+ split_keys = {k.split(self._delimiter)[0] for k in keys}
+ for k in sorted(split_keys, key=lambda x: int(x)):
+ if subset[k].original_type == tuple:
+ out.append(tuple(self._child_as_list(pk, k)))
+ elif subset[k].original_type == list:
+ out.append(self._child_as_list(pk, k))
+ elif subset[k].original_type == set:
+ out.append(set(self._child_as_list(pk, k)))
+ elif subset[k].original_type == dict:
+ out.append(subset[k].as_dict())
+ return out
+
+ # Python prior 3.6 does not guarantee insertion order, remove it after
+ # EOL python 3.5 - 2020-09-13
+ if sys.version_info[0:2] < (3, 6): # pragma: nocover
+ return [subset[k] for k in sorted(keys, key=lambda x: int(x))]
+ else:
+ return [subset[k] for k in keys]
diff --git a/actions/devel/psql/nuvolaris/config.py b/actions/devel/psql/nuvolaris/config.py
new file mode 100644
index 0000000..787d4a7
--- /dev/null
+++ b/actions/devel/psql/nuvolaris/config.py
@@ -0,0 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import flatdict, json, os
+import logging
+
+_config = {}
+
+# define a configuration
+# the configuration is a map, followed by a list of labels
+# the map can be a serialized json and will be flattened to a map of values.
+# you can have only a configuration active at a time
+# if you want to set a new configuration you have to clean it
+def configure(spec: dict):
+ global _config
+ _config = dict(flatdict.FlatDict(spec, delimiter="."))
+
+ #forces autodetect of kube if not provided
+ if not exists('nuvolaris.kube'):
+ put('nuvolaris.kube','auto')
+
+ return True
+
+def clean():
+ global _config
+ _config = {}
+
+def exists(key):
+ return key in _config
+
+def get(key, envvar=None, defval=None):
+ val = _config.get(key)
+
+ if envvar and envvar in os.environ:
+ val = os.environ[envvar]
+
+ if val:
+ return val
+
+ return defval
+
+def put(key, value):
+ _config[key] = value
+ return True
+
+def delete(key):
+ if key in _config:
+ del _config[key]
+ return True
+
+def getall(prefix=""):
+ res = {}
+ for key in _config.keys():
+ if key.startswith(prefix):
+ res[key] = _config[key]
+ return res
+
+def keys(prefix=""):
+ res = []
+ if _config:
+ for key in _config.keys():
+ if key.startswith(prefix):
+ res.append(key)
+ return res
+
+def detect_labels(labels=None):
+ # skip autodetection of nuvolaris.kube if already available in _config
+ if exists('nuvolaris.kube') and get('nuvolaris.kube') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.kube={get('nuvolaris.kube')}")
+ return {}
+
+ # read labels if not available
+ if not labels:
+ import nuvolaris.kube as kube
+ labels = kube.kubectl("get", "nodes", jsonpath='{.items[].metadata.labels}')
+
+ res = {}
+ kube = None
+ for i in labels:
+ for j in list(i.keys()):
+ if j.startswith("nuvolaris.io/"):
+ key = f"nuvolaris.{j[13:]}"
+ res[key] = i[j]
+ _config[key] = i[j]
+
+ for i in labels:
+ for j in list(i.keys()):
+ # detect the kube type
+ if j.find("eksctl.io") >= 0:
+ kube ="eks"
+ break
+ if j.find("k8s.io/cloud-provider-aws") >= 0:
+ kube ="eks"
+ break
+ elif j.find("microk8s.io") >= 0:
+ kube = "microk8s"
+ break
+ elif j.find("lke.linode.com") >=0:
+ kube = "lks"
+ break
+ elif j.find("node.openshift.io") >=0:
+ kube = "openshift"
+ break
+ elif j.endswith("kubernetes.io/instance-type") and i[j] == "k3s":
+ kube = "k3s"
+ break
+ elif j.find("cloud.google.com/gke") >=0:
+ kube = "gke"
+ break
+ elif j.find("kubernetes.azure.com") >=0:
+ kube = "aks"
+ break
+ # assign all the 'nuvolaris.io' labels
+
+ if kube:
+ res["nuvolaris.kube"] = kube
+ _config["nuvolaris.kube"] = kube
+ return res
+
+ # defaults to generic if it is not yet detected
+ if exists('nuvolaris.kube') and get('nuvolaris.kube') == 'auto':
+ _config["nuvolaris.kube"] = "generic"
+ res["nuvolaris.kube"] = "generic"
+
+ return res
+
+def detect_storage(storages=None):
+ res = {}
+ if not storages:
+ import nuvolaris.util as util
+
+ try:
+ detect_storageclass = True
+ detect_provisioner = True
+
+ # skips autodetection if already provided
+ if exists('nuvolaris.storageclass') and get('nuvolaris.storageclass') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.storageclass={get('nuvolaris.storageclass')}")
+ detect_storageclass = False
+
+ if exists('nuvolaris.provisioner') and get('nuvolaris.provisioner') != 'auto':
+ logging.info(f"*** configuration provided already a nuvolaris.provisioner={get('nuvolaris.provisioner')}")
+ detect_provisioner = False
+
+ if not detect_storageclass and not detect_provisioner:
+ return res
+
+ if detect_storageclass:
+ storage_class = util.get_default_storage_class()
+ if storage_class:
+ res['nuvolaris.storageclass'] = storage_class
+ _config['nuvolaris.storageclass'] = storage_class
+
+ if detect_provisioner:
+ provisioner = util.get_default_storage_provisioner()
+ if(provisioner):
+ res['nuvolaris.provisioner'] = provisioner
+ _config['nuvolaris.provisioner'] = provisioner
+ except:
+ pass
+ return res
+
+def detect_env():
+ _config['operator.image'] = os.environ.get("OPERATOR_IMAGE", "missing-OPERATOR_IMAGE")
+ _config['operator.tag'] = os.environ.get("OPERATOR_TAG", "missing-OPERATOR_TAG")
+
+ # skip autodetection of controller.image if already configure into whisk.yaml
+ if not exists('controller.image'):
+ _config['controller.image'] = os.environ.get("CONTROLLER_IMAGE", "missing-CONTROLLER_IMAGE")
+ _config['controller.tag'] = os.environ.get("CONTROLLER_TAG", "missing-CONTROLLER_TAG")
+ else:
+ logging.warn(f"OW controller image detection skipped. Using {get('controller.image')}")
+
+ if not exists('invoker.image'):
+ _config['invoker.image'] = os.environ.get("INVOKER_IMAGE", "missing-INVOKER_IMAGE")
+ _config['invoker.tag'] = os.environ.get("INVOKER_TAG", "missing-INVOKER_TAG")
+ else:
+ logging.warn(f"OW invoker image detection skipped. Using {get('invoker.image')}")
+
+def detect():
+ detect_storage()
+ detect_labels()
+ detect_env()
+
+def dump_config():
+ import nuvolaris.config as cfg
+ for k in cfg.getall():
+ logging.debug(f"{k} = {cfg.get(k)}")
diff --git a/actions/devel/psql/nuvolaris/couchdb_util.py b/actions/devel/psql/nuvolaris/couchdb_util.py
new file mode 100644
index 0000000..3adb6b9
--- /dev/null
+++ b/actions/devel/psql/nuvolaris/couchdb_util.py
@@ -0,0 +1,164 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import os, json, time, sys, logging
+import requests as req
+import nuvolaris.config as cfg
+
+class CouchDB:
+ def __init__(self):
+ self.db_protocol = "http"
+ self.db_prefix = "nuvolaris_"
+ self.db_port = cfg.get("couchdb.port", "COUCHDB_SERVICE_PORT", "5984")
+ self.db_host = cfg.get("couchdb.host", "COUCHDB_SERVICE_HOST", "couchdb")
+ self.db_username = cfg.get("couchdb.admin.user", "COUCHDB_ADMIN_USER", "whisk_admin")
+ self.db_password = cfg.get("couchdb.admin.password", "COUCHDB_ADMIN_PASSWORD", "some_passw0rd")
+ self.db_auth = req.auth.HTTPBasicAuth(self.db_username,self.db_password)
+ self.db_url = f"{self.db_protocol}://{self.db_host}:{self.db_port}"
+ self.db_base = f"{self.db_url}/{self.db_prefix}"
+ self.db_session = req.Session()
+ self.db_session.auth = self.db_auth
+
+ def wait_db_ready(self, max_seconds):
+ logging.info("entering CouchDB.wait_db_ready()")
+ start = time.time()
+ delta = 0
+ session = req.Session()
+ while delta < max_seconds:
+ try:
+ r = session.get(f"{self.db_url}/_utils", timeout=5)
+ logging.info(f"CouchDB.wait_db_ready() got response code = {r.status_code}")
+ if r.status_code == 200:
+ return True
+ except Exception as e:
+ logging.info(f"waiting since: {delta} seconds")
+ delta = int(time.time() - start)
+ time.sleep(1)
+ return False
+
+ # check if database exists, return boolean
+ def check_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.head(url)
+ return r.status_code == 200
+
+ # delete database, return true if ok
+ def delete_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.delete(url)
+ return r.status_code == 200
+
+ # create db, return true if ok
+ def create_db(self, database):
+ url = f"{self.db_base}{database}"
+ r = self.db_session.put(url)
+ return r.status_code == 201
+
+ # database="subjects"
+ def recreate_db(self, database, recreate=False):
+ msg = "recreate_db:"
+ exists = self.check_db(database)
+ if recreate and exists:
+ msg += " deleted"
+ self.delete_db(database)
+ if recreate or not exists:
+ msg += " created"
+ self.create_db(database)
+ return msg
+
+ def get_doc(self, database, id, user=None, password="", no_auth=False):
+ url = f"{self.db_base}{database}/{id}"
+ session = req.Session()
+ if no_auth:
+ db_auth=None
+ elif user:
+ db_auth=req.auth.HTTPBasicAuth(user, password)
+ else:
+ db_auth = self.db_auth
+ session.auth = db_auth
+ r = session.get(url)
+ if r.status_code == 200:
+ return json.loads(r.text)
+ return None
+
+ def update_doc(self, database, doc):
+ if '_id' in doc:
+ url = f"{self.db_base}{database}/{doc['_id']}"
+ cur = self.get_doc(database, doc['_id'])
+ if cur and '_rev' in cur:
+ doc['_rev'] = cur['_rev']
+ r = self.db_session.put(url, json=doc)
+ else:
+ r = self.db_session.put(url, json=doc)
+ return r.status_code in [200,201]
+ return False
+
+ def delete_doc(self, database, id):
+ cur = self.get_doc(database, id)
+ if cur and '_rev' in cur:
+ url = f"{self.db_base}{database}/{cur['_id']}?rev={cur['_rev']}"
+ r = self.db_session.delete(url)
+ return r.status_code == 200
+ return False
+
+ def configure_single_node(self):
+ url = f"{self.db_url}/_cluster_setup"
+ data = {"action": "enable_single_node", "singlenode": True, "bind_address": "0.0.0.0", "port": 5984}
+ r = self.db_session.post(url, json=data)
+ return r.status_code == 201
+
+ def configure_no_reduce_limit(self):
+ url = f"{self.db_url}/_node/_local/_config/query_server_config/reduce_limit"
+ data=b'"false"'
+ r = self.db_session.put(url, data=data)
+ return r.status_code == 200
+
+ def add_user(self, username: str, password: str):
+ userpass = {"name": username, "password": password, "roles": [], "type": "user"}
+ url = f"{self.db_url}/_users/org.couchdb.user:{username}"
+ res = self.db_session.put(url, json=userpass)
+ return res.status_code in [200, 201, 421]
+
+ #def add_role(self, database: str, members: list[str] = [], admins: list[str] =[]):
+ def add_role(self, database: str, members = [], admins =[]):
+ roles = {"admins": { "names": admins, "roles": [] }, "members": { "names": members, "roles": [] } }
+ url = f"{self.db_base}{database}/_security"
+ res = self.db_session.put(url, json=roles)
+ return res.status_code in [200, 201, 421]
+
+#
+# Submit a POST request to the _find endpoint using the specified selector
+#
+ def find_doc(self, database, selector, user=None, password="", no_auth=False):
+ url = f"{self.db_base}{database}/_find"
+ headers = {'Content-Type': 'application/json'}
+ session = req.Session()
+
+ if no_auth:
+ db_auth=None
+ elif user:
+ db_auth=req.auth.HTTPBasicAuth(user, password)
+ else:
+ db_auth = self.db_auth
+
+ session.auth=db_auth
+ r = session.post(url, headers=headers, data=selector)
+ if r.status_code == 200:
+ return json.loads(r.text)
+
+ logging.warn(f"query to {url} failed with {r.status_code}. Body {r.text}")
+ return None
\ No newline at end of file
diff --git a/actions/devel/redis/__main__.py b/actions/devel/redis/__main__.py
new file mode 100644
index 0000000..e183fdc
--- /dev/null
+++ b/actions/devel/redis/__main__.py
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import json
+
+from common.authorize import Authorize
+from common.command_data import CommandData
+from command.redis import Redis
+from base64 import b64decode
+
+class ApiError(Exception):
+ pass
+
+def build_error(message: str):
+ return {
+ "statusCode": 400,
+ "body": message
+ }
+
+def build_response(data:CommandData):
+ meta_data = data.get_metadata()
+ return {
+ "statusCode": meta_data['status'],
+ "body": meta_data['result']
+ }
+
+def parse_body(args):
+ try:
+ return b64decode(args['__ow_body']).decode().strip()
+ except Exception as e:
+ print(e)
+ raise ApiError("could not parse __ow_body as base64")
+
+def main(args):
+ """
+ Action implementing a generic command wrapper for the nuv devel plugin. The action must be called with a POST request receiving a JSON
+ payload similar to this one
+ {
+ "command":"SET nuvolaris:key1 donald_duck"
+ }
+
+ the invoker must provide a x-impersonate-auth header containing the Openwhisk BASIC authentication of the wsku/user the action should impersonate
+ when submitting the command. Every specific provider will support specific request type and command.
+ WARNING: the body will be received as base64 encoded string as this action will be deployed with --web raw enabled flag
+ """
+ headers = args['__ow_headers']
+ if('x-impersonate-auth' not in headers):
+ return build_error("invalid request, missing mandatory header: x-impersonate-auth")
+
+ if(len(args['__ow_body']) == 0):
+ return build_error("invalid request, no command payload received")
+
+ try:
+ user_data = Authorize(args['couchdb_host'],args['couchdb_user'],args['couchdb_password']).login(headers['x-impersonate-auth'])
+ cmd = CommandData(json.loads(parse_body(args)))
+ return build_response(Redis(user_data).execute(cmd))
+ except Exception as e:
+ return build_error(f"failed to execute nuv devel redis. Reason: {str(e)}")
\ No newline at end of file
diff --git a/actions/devel/redis/command/redis.py b/actions/devel/redis/command/redis.py
new file mode 100644
index 0000000..9fa4a9e
--- /dev/null
+++ b/actions/devel/redis/command/redis.py
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import common.util as ut
+import redis
+
+from common.command_data import CommandData
+
+class Redis():
+ """
+ Implementation of a Redis Command executor. It will require
+ a user_data dictionary linked to a specific user.
+ """
+
+ def __init__(self, user_data):
+ self._user_data = user_data
+ self._redis_prefix = ut.get_env_value(user_data,"REDIS_PREFIX")
+ self._redis_url = ut.get_env_value(user_data,"REDIS_URL")
+ self.validate()
+
+ def validate(self):
+ """
+ Validate that the provided user_data contains the appropriate
+ metadata for being able to submit a redis command.
+ """
+ if not self._redis_prefix or not self._redis_url:
+ raise Exception("user does not have valid REDIS environment set")
+
+ def execute(self, input:CommandData):
+ print(f"**** Redis command to execute {input.command()}")
+ try:
+ r = redis.from_url(self._redis_url)
+ result = r.execute_command(input.command())
+ input.result(str(result.decode('utf-8')))
+ input.status(200)
+ except Exception as e:
+ input.result(f"could not execute redis command {e}")
+ input.status(400)
+
+ return input
\ No newline at end of file
diff --git a/actions/devel/redis/common/authorize.py b/actions/devel/redis/common/authorize.py
new file mode 100644
index 0000000..f7d716f
--- /dev/null
+++ b/actions/devel/redis/common/authorize.py
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+from base64 import b64decode, b64encode
+from urllib.parse import quote, unquote
+
+import nuvolaris.config as cfg
+import nuvolaris.couchdb_util as cu
+import json
+import common.util as ut
+
+USER_META_DBN = "users_metadata"
+SUBJECT_META_DBN = "subjects"
+
+class DecodeError(Exception):
+ pass
+
+class EncodeError(Exception):
+ pass
+
+class AuthorizationError(Exception):
+ pass
+
+class Authorize():
+
+ def __init__(self, cdb_host, cdb_user, cdb_pwd):
+ cfg.clean()
+ cfg.put("couchdb.host", cdb_host)
+ cfg.put("couchdb.admin.user", cdb_user)
+ cfg.put("couchdb.admin.password", cdb_pwd)
+ self._db = cu.CouchDB()
+
+ def encode(self,username, password):
+ """Returns an HTTP basic authentication encrypted string given a valid
+ username and password.
+ """
+ if ':' in username:
+ raise EncodeError
+
+ username_password = f'{quote(username)}:{quote(password)}'
+ return f'Basic {b64encode(username_password.encode()).decode()}'
+
+ def _parse_b64(self, encoded_str):
+ try:
+ username, password = b64decode(encoded_str).decode().split(':', 1)
+ except:
+ # fallback in case the token is not bas64 encoded
+ username, password = encoded_str.split(':', 1)
+
+ if not username or not password:
+ raise DecodeError("authentication token does not seems to be b64 encoded")
+
+ return username, password
+
+ def decode(self, encoded_str):
+ """Decode an encrypted HTTP basic authentication string. Returns a tuple of
+ the form (username, password), and raises a DecodeError exception if
+ nothing could be decoded.
+ """
+ split = encoded_str.strip().split(' ')
+
+ # If split is only one element, try to decode the username and password
+ # directly.
+ if len(split) == 1:
+ username, password = self._parse_b64(split[0])
+
+ # If there are only two elements, check the first and ensure it says
+ # 'basic' so that we know we're about to decode the right thing. If not,
+ # bail out.
+ elif len(split) == 2:
+ if split[0].strip().lower() == 'basic':
+ username, password = self._parse_b64(split[1])
+ else:
+ raise DecodeError("authentication token provides more than 2 elements. could not parse properly")
+
+ # If there are more than 2 elements, something crazy must be happening.
+ # Bail.
+ else:
+ raise DecodeError("unpredictable error parsing authentication token")
+
+ return unquote(username), unquote(password)
+
+ def fetch_subject(self, uuid: str, key: str):
+ """
+ Query the internal couchdb searching for the subject mathing the given uuid, key.
+ Normally these stored in wsk or wsku in the form uuid:key
+ :param uuid the OW subject uuid
+ :param key the OW subject key
+ :return a ubject document
+ """
+ print(f"searching for openwhisk subject {uuid}")
+ try:
+ selector= {
+ "selector": {"namespaces": {"$elemMatch": {"uuid": uuid,"key": key}}}
+ }
+
+ response = self._db.find_doc(SUBJECT_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris namespace for user {uuid} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {uuid} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {uuid}. Reason: {e}")
+ return None
+
+ def fetch_user_data(self, username: str):
+ """
+ Query the internal couchdb searching for the given principal to retrieve all the
+ relevant metadata
+ """
+ print(f"searching for user {username} meta-data")
+ try:
+ selector = {"selector":{"login": {"$eq": username }}}
+ response = self._db.find_doc(USER_META_DBN, json.dumps(selector))
+
+ if(response['docs']):
+ docs = list(response['docs'])
+ if(len(docs) > 0):
+ print(f"Nuvolaris metadata for user {username} found. Returning Result.")
+ return docs[0]
+
+ print(f"Nuvolaris metadata for user {username} not found!")
+ return None
+ except Exception as e:
+ print(f"failed to query Nuvolaris metadata for user {username}. Reason: {e}")
+ return None
+
+ def login(self, authorization: str):
+ """
+ Attempt to login the user identified by the given Openwhisk authorization AUTH token as base64
+ """
+ uuid, key = self.decode(authorization)
+ subject = self.fetch_subject(uuid, key)
+
+ if not subject:
+ raise AuthorizationError("Openwhisk subject not found.")
+
+ user_data = self.fetch_user_data(subject['subject'])
+
+ if user_data:
+ return user_data
+
+ raise AuthorizationError("Could not retrieve user metadata.")
\ No newline at end of file
diff --git a/actions/devel/redis/common/command_data.py b/actions/devel/redis/common/command_data.py
new file mode 100644
index 0000000..44f3faa
--- /dev/null
+++ b/actions/devel/redis/common/command_data.py
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
... 62333 lines suppressed ...