You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@submarine.apache.org by cd...@apache.org on 2023/01/07 00:56:37 UTC
[submarine] branch master updated: SUBMARINE-1352. Move submarine-cloud-v2 to submarine-cloud-v3
This is an automated email from the ASF dual-hosted git repository.
cdmikechen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new a7e2c545 SUBMARINE-1352. Move submarine-cloud-v2 to submarine-cloud-v3
a7e2c545 is described below
commit a7e2c5458bba1ebfb78f979b56add77f08f65184
Author: cdmikechen <cd...@apache.org>
AuthorDate: Sat Jan 7 07:59:36 2023 +0800
SUBMARINE-1352. Move submarine-cloud-v2 to submarine-cloud-v3
### What is this PR for?
Move submarine-operator from v2 to v3.
### What type of PR is it?
Feature
### Todos
* [x] - Remove v2 cicd
* [x] - Optimize v3 cicd
* [x] - Move image build to v3
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-1352
### How should this be tested?
CICD
### Screenshots (if appropriate)
NA
### Questions:
* Do the license files need updating? No
* Are there breaking changes for older versions? Yes
* Does this need new documentation? No
Author: cdmikechen <cd...@apache.org>
Signed-off-by: cdmikechen <cd...@apache.org>
Closes #1031 from cdmikechen/SUBMARINE-1352 and squashes the following commits:
d1cfd2ff [cdmikechen] Remove v2 cicd
49949265 [cdmikechen] Move v2 to v3
---
.github/workflows/deploy_docker_images.yml | 2 +-
.github/workflows/master.yml | 90 ++--------------------
.gitignore | 4 +
dev-support/style-check/lint-go.sh | 2 +-
submarine-cloud-v3/Makefile | 13 +++-
.../crd/bases/submarine.apache.org_submarines.yaml | 2 +-
submarine-cloud-v3/config/rbac/role.yaml | 2 +-
submarine-cloud-v3/docs/developer-guide.md | 4 +
.../hack/boilerplate.yaml.txt | 11 ---
submarine-cloud-v3/hack/verify-codegen.sh | 53 +++++++++++++
10 files changed, 82 insertions(+), 101 deletions(-)
diff --git a/.github/workflows/deploy_docker_images.yml b/.github/workflows/deploy_docker_images.yml
index 3fa0f010..849f4916 100644
--- a/.github/workflows/deploy_docker_images.yml
+++ b/.github/workflows/deploy_docker_images.yml
@@ -76,7 +76,7 @@ jobs:
run: docker push apache/submarine:jupyter-notebook-gpu-$SUBMARINE_VERSION
- name: Build submarine operator
- run: ./dev-support/docker-images/operator/build.sh
+ run: ./dev-support/docker-images/operator-v3/build.sh
- name: Push submarine-operator docker image
run: docker push apache/submarine:operator-$SUBMARINE_VERSION
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index ab8c7ac8..14434571 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -32,7 +32,7 @@ jobs:
- id: set-matrix
run: |
echo "::set-output name=matrix::[\"v1.19.16\", \"v1.20.15\", \"v1.21.14\"]"
- submarine-operator-verify-codegen:
+ submarine-operator-verify:
runs-on: ubuntu-latest
timeout-minutes: 10
env:
@@ -47,93 +47,17 @@ jobs:
with:
go-version: "1.17.2"
- name: Check golang version
- working-directory: ${{ env.working-directory }}/submarine-cloud-v2
+ working-directory: ${{ env.working-directory }}/submarine-cloud-v3
run: go version
- name: Install Dependencies
- working-directory: ${{ env.working-directory }}/submarine-cloud-v2
+ working-directory: ${{ env.working-directory }}/submarine-cloud-v3
run: go mod vendor
- name: Verify Codegen
- working-directory: ${{ env.working-directory }}/submarine-cloud-v2
+ working-directory: ${{ env.working-directory }}/submarine-cloud-v3
run: ./hack/verify-codegen.sh
- submarine-operator-e2e-test:
- needs: generate-k8s-versions-array
- runs-on: ubuntu-latest
- timeout-minutes: 60
- strategy:
- matrix:
- k8s-version: ${{fromJSON(needs.generate-k8s-versions-array.outputs.matrix)}}
- fail-fast: false
- steps:
- - uses: actions/checkout@v2
- with:
- fetch-depth: 50
- - name: Set up JDK 11
- uses: actions/setup-java@v1
- with:
- java-version: "11"
- - name: Set up Maven 3.6.3
- uses: stCarolas/setup-maven@v4
- with:
- maven-version: 3.6.3
- - name: Setup Golang 1.17.2
- uses: actions/setup-go@v2
- with:
- go-version: "1.17.2"
- - uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-
- - name: Check version
- run: |
- mvn --version
- java -version
- go version
- helm version
- kind version
- - name: Create kind cluster
- run: kind create cluster --config ./.github/config/kind-config-kind.yaml --wait 3m --image kindest/node:${{ matrix.k8s-version }}
- - name: Download Istio 1.13.0
- run: |
- curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.13.0 sh -
- cd istio-1.13.0
- echo "$PWD/bin" >> $GITHUB_PATH
- - name: Install Istio
- run: istioctl install -y --set values.global.proxy.resources.requests.cpu=10m
- - name: Show K8s cluster information
- run: |
- kubectl cluster-info
- kubectl version
- kubectl get pods -n kube-system
- echo "current-context:" $(kubectl config current-context)
- echo "environment-kubeconfig:" ${KUBECONFIG}
- - name: Build Image locally
- run: .github/scripts/build-image-locally.sh
- - name: Install Golang Dependencies
- working-directory: submarine-cloud-v2
- run: go mod vendor
- - name: Install Helm Dependencies
- run: |
- sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
- sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
- helm dependency update ./helm-charts/submarine
- helm install --wait --set dev=true --set storageClass.provisioner=rancher.io/local-path --set storageClass.volumeBindingMode=WaitForFirstConsumer submarine ./helm-charts/submarine
- - name: Run end-to-end test
- working-directory: submarine-cloud-v2
- run: go test -v -timeout 30m ./test/e2e
- - name: Failure status
- run: |
- kubectl describe nodes
- kubectl get all -A
- kubectl get events -A --sort-by='{.lastTimestamp}'
- kubectl describe submarine -A
- if: ${{ failure() }}
- - name: Delete temporary build artifacts before caching
- run: |
- #Never cache local artifacts
- rm -rf ~/.m2/repository/org/apache/submarine
- if: always()
+ - name: Verify fmt and vet
+ working-directory: ${{ env.working-directory }}/submarine-cloud-v3
+ run: make fmt && make vet
submarine-operator-v3-e2e-test:
needs: generate-k8s-versions-array
runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index f45e20ed..689e1c46 100644
--- a/.gitignore
+++ b/.gitignore
@@ -89,6 +89,10 @@ submarine-cloud-v2/helm-charts/submarine-operator/charts/*
# add operator docker build tmp
dev-support/docker-images/operator/tmp/
+# submarine-cloud-v3
+submarine-cloud-v3/vendor/*
+submarine-cloud-v3/_tmp/*
+
# vscode file
.project
.classpath
diff --git a/dev-support/style-check/lint-go.sh b/dev-support/style-check/lint-go.sh
index 9205d8ee..62141911 100755
--- a/dev-support/style-check/lint-go.sh
+++ b/dev-support/style-check/lint-go.sh
@@ -18,7 +18,7 @@
#
cd "$(dirname $0)"
-cd ../../submarine-cloud-v2
+cd ../../submarine-cloud-v3
if [ -n "$(go fmt ./...)" ]; then
echo "Go code is not formatted, please run 'go fmt ./...'." >&2
diff --git a/submarine-cloud-v3/Makefile b/submarine-cloud-v3/Makefile
index 7f4b6ad2..bf42a1b8 100644
--- a/submarine-cloud-v3/Makefile
+++ b/submarine-cloud-v3/Makefile
@@ -81,6 +81,11 @@ endif
SHELL = /usr/bin/env bash -o pipefail
.SHELLFLAGS = -ec
+# We define a FAIL_ON_STDOUT awk command that detects if an error message is output,
+# which we will use several times later.
+# NR is a built-in variable for the number of record.
+FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1 } }'
+
.PHONY: all
all: build
@@ -104,8 +109,10 @@ help: ## Display this help.
##@ Development
.PHONY: manifests
+# controller-gen does not add headerFile to the yaml, so we need to add it manually for now
+# https://github.com/kubernetes-sigs/controller-tools/issues/464
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
- $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+ $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases && sed -i '' '1r ./hack/boilerplate.yaml.txt' config/crd/bases/*.yaml && sed -i '' '1r ./hack/boilerplate.yaml.txt' config/rbac/role.yaml
.PHONY: generate
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -113,11 +120,11 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and
.PHONY: fmt
fmt: ## Run go fmt against code.
- go fmt ./...
+ go fmt ./... 2>&1 | $(FAIL_ON_STDOUT)
.PHONY: vet
vet: ## Run go vet against code.
- go vet ./...
+ go vet ./... 2>&1 | $(FAIL_ON_STDOUT)
.PHONY: test
test: manifests generate fmt vet envtest ## Run tests.
diff --git a/submarine-cloud-v3/config/crd/bases/submarine.apache.org_submarines.yaml b/submarine-cloud-v3/config/crd/bases/submarine.apache.org_submarines.yaml
index 1dd70ed1..b9a5f7ba 100644
--- a/submarine-cloud-v3/config/crd/bases/submarine.apache.org_submarines.yaml
+++ b/submarine-cloud-v3/config/crd/bases/submarine.apache.org_submarines.yaml
@@ -1,3 +1,4 @@
+---
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -15,7 +16,6 @@
# limitations under the License.
#
----
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
diff --git a/submarine-cloud-v3/config/rbac/role.yaml b/submarine-cloud-v3/config/rbac/role.yaml
index cdb8c456..7fd9976a 100644
--- a/submarine-cloud-v3/config/rbac/role.yaml
+++ b/submarine-cloud-v3/config/rbac/role.yaml
@@ -1,3 +1,4 @@
+---
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -15,7 +16,6 @@
# limitations under the License.
#
----
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
diff --git a/submarine-cloud-v3/docs/developer-guide.md b/submarine-cloud-v3/docs/developer-guide.md
index 706b4eff..f3a9c200 100644
--- a/submarine-cloud-v3/docs/developer-guide.md
+++ b/submarine-cloud-v3/docs/developer-guide.md
@@ -190,6 +190,10 @@ eval $(minikube docker-env -u)
For `go` files, please use [gofmt](https://golang.org/pkg/cmd/gofmt/) to format the code.
+```bash
+make fmt
+```
+
For `yaml` files, please use [prettier](https://prettier.io/) to format the code.
## Generate API
diff --git a/dev-support/style-check/lint-go.sh b/submarine-cloud-v3/hack/boilerplate.yaml.txt
old mode 100755
new mode 100644
similarity index 77%
copy from dev-support/style-check/lint-go.sh
copy to submarine-cloud-v3/hack/boilerplate.yaml.txt
index 9205d8ee..555eccfe
--- a/dev-support/style-check/lint-go.sh
+++ b/submarine-cloud-v3/hack/boilerplate.yaml.txt
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -17,12 +15,3 @@
# limitations under the License.
#
-cd "$(dirname $0)"
-cd ../../submarine-cloud-v2
-
-if [ -n "$(go fmt ./...)" ]; then
- echo "Go code is not formatted, please run 'go fmt ./...'." >&2
- exit 1
-else
- echo "Go code is formatted"
-fi
diff --git a/submarine-cloud-v3/hack/verify-codegen.sh b/submarine-cloud-v3/hack/verify-codegen.sh
new file mode 100755
index 00000000..791db741
--- /dev/null
+++ b/submarine-cloud-v3/hack/verify-codegen.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd ${SCRIPT_ROOT}
+
+# Check api
+DIFFROOT="./api"
+TMP_DIFFROOT="./_tmp/api"
+_tmp="./_tmp"
+
+cleanup() {
+ rm -rf "${_tmp}"
+}
+trap "cleanup" EXIT SIGINT
+
+cleanup
+
+mkdir -p "${TMP_DIFFROOT}"
+cp -a "${DIFFROOT}"/* "${TMP_DIFFROOT}"
+
+make generate
+echo "diffing ${DIFFROOT} against freshly generated codegen"
+ret=0
+diff -Naupr "${DIFFROOT}" "${TMP_DIFFROOT}" || ret=$?
+cp -a "${TMP_DIFFROOT}"/* "${DIFFROOT}"
+if [[ $ret -eq 0 ]]
+then
+ echo "${DIFFROOT} up to date."
+else
+ echo "${DIFFROOT} is out of date. Please run 'make generate'"
+ exit 1
+fi
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@submarine.apache.org
For additional commands, e-mail: dev-help@submarine.apache.org