You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by kl...@apache.org on 2023/10/27 08:56:46 UTC

(incubator-devlake) 01/01: refactor: simplify Makefile

This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch kw-makefile
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git

commit 1ffdef507ef260a7b06f7840f0452f57fd6656c9
Author: Klesh Wong <zh...@merico.dev>
AuthorDate: Fri Oct 27 16:56:36 2023 +0800

    refactor: simplify Makefile
---
 .github/workflows/test-e2e.yml                     |  4 +-
 backend/Makefile                                   | 68 +++++-----------------
 .../{compile-plugins.sh => build-plugins.sh}       | 17 ++++--
 backend/scripts/build-python.sh                    | 23 ++++++++
 backend/scripts/build-server.sh                    | 28 +++++++++
 backend/scripts/e2e-test-go-plugins.sh             | 33 +++++++++++
 backend/scripts/e2e-test.sh                        | 34 +++++++++++
 backend/scripts/jira-api-benchmark.sh              |  0
 backend/scripts/unit-test-go.sh                    | 24 ++++++++
 9 files changed, 170 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
index 59619efae..4b078f8f6 100644
--- a/.github/workflows/test-e2e.yml
+++ b/.github/workflows/test-e2e.yml
@@ -73,7 +73,7 @@ jobs:
           go version
           cp env.example .env
           cd backend
-          make e2e-plugins-test
+          make e2e-test-go-plugins
           make e2e-test
 
   e2e-postgres:
@@ -113,5 +113,5 @@ jobs:
           go version
           cp env.example .env
           cd backend
-          make e2e-plugins-test
+          make e2e-test-go-plugins
           make e2e-test
diff --git a/backend/Makefile b/backend/Makefile
index 8f9ba6bd9..908f29c95 100644
--- a/backend/Makefile
+++ b/backend/Makefile
@@ -23,6 +23,9 @@ IMAGE_REPO ?= "apache"
 VERSION = $(TAG)@$(SHA)
 PYTHON_DIR ?= "./python"
 
+
+all: build
+
 go-dep:
 	go install github.com/vektra/mockery/v2@v2.20.0
 	go install github.com/swaggo/swag/cmd/swag@v1.8.12
@@ -38,30 +41,16 @@ swag:
 	scripts/swag.sh
 
 build-plugin:
-	if [ "$(PLUGIN)" = "none" ]; then \
-		echo "Building plugins will be skipped"; \
-	elif [ "$(DEBUG)" = "true" ]; then \
-		PLUGIN=$(PLUGIN) sh scripts/compile-plugins.sh -gcflags='all=-N -l'; \
-	else \
-		PLUGIN=$(PLUGIN) sh scripts/compile-plugins.sh; \
-	fi
+	scripts/build-plugins.sh
 
 build-server: swag
-	if [ "$(DEBUG)" = "true" ]; then \
-		go build -gcflags='all=-N -l' -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/; \
-	else \
-		go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/; \
-	fi
+	scripts/build-server.sh
 
 build-python: #don't mix this with the other build commands
-	find ./python/ -name "*.sh" | xargs chmod +x &&\
-	sh python/build.sh
-	sh python/build.sh python/test
+	scripts/build-python.sh
 
 build: build-plugin build-server
 
-all: build
-
 run:
 	go run server/main.go
 
@@ -80,50 +69,23 @@ mock:
 
 test: unit-test e2e-test
 
-unit-test: mock unit-test-only python-unit-test
+unit-test: mock unit-test-go unit-test-python
 
-unit-test-only:
-	set -e;\
-	for m in $$(go list ./... | egrep -v 'test|models|e2e'); do \
-		echo $$m; go test -timeout 60s -v $$m || exit $$?; \
-	done; \
+unit-test-go:
+	scripts/unit-test-go.sh
 
 build-pydevlake:
 	poetry install -C python/pydevlake
 
-python-unit-test: build-pydevlake
+unit-test-python: build-pydevlake
 	sh python/build.sh python/test &&\
 	sh ./python/run_tests.sh
 
-e2e-plugins-test:
-	export TZ=UTC;\
-	export ENV_PATH=$(shell readlink -f .env);\
-	set -e;\
-	for m in $$(go list ./plugins/... | egrep 'e2e'); do \
-		echo $$m; go test -timeout 300s -gcflags=all=-l -v $$m || exit $$?; \
-	done; \
-
-e2e-test-init:
-	export ENV_PATH=$(shell readlink -f .env);\
-	set -e;\
-	go run ./test/init.go || exit $$?;\
-
-e2e-test-run:
-	export ENV_PATH=$(shell readlink -f .env);\
-	set -e;\
-	for m in $$(go list ./test/e2e/... | grep -v manual); do \
-		echo $$m; go test -failfast -timeout 120s -v $$m || exit $$?; \
-	done; \
-
-e2e-test: e2e-test-init e2e-test-run
-
-integration-test:
-	export ENV_PATH=$(shell readlink -f .env);\
-	set -e;\
-	go run ./test/init.go || exit $$?;\
-	for m in $$(go list ./test/integration/...); do \
-		echo $$m; go test -timeout 300s -v $$m || exit $$?; \
-	done; \
+e2e-test-go-plugins:
+	scripts/e2e-test-go-plugins.sh
+
+e2e-test:
+	scripts/e2e-test.sh
 
 lint:
 	golangci-lint run
diff --git a/backend/scripts/compile-plugins.sh b/backend/scripts/build-plugins.sh
old mode 100644
new mode 100755
similarity index 85%
rename from backend/scripts/compile-plugins.sh
rename to backend/scripts/build-plugins.sh
index 4ee940d3f..d192edf98
--- a/backend/scripts/compile-plugins.sh
+++ b/backend/scripts/build-plugins.sh
@@ -34,29 +34,34 @@ echo "Usage: "
 echo "  build all plugins:              $0 [golang build flags...]"
 echo "  build and keep specified plugins only: PLUGIN=github,jira $0 [golang build flags...]"
 
-SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
-PLUGIN_SRC_DIR=$SCRIPT_DIR/../plugins
-PLUGIN_OUTPUT_DIR=$SCRIPT_DIR/../bin/plugins
+ROOT_DIR=$(dirname $(dirname "$0"))
+EXTRA=""
+PLUGIN_SRC_DIR=$ROOT_DIR/plugins
+PLUGIN_OUTPUT_DIR=${PLUGIN_DIR:-$ROOT_DIR/bin/plugins}
 
+if [ -n "$DEVLAKE_DEBUG" ]; then
+    EXTRA="-gcflags='all=-N -l'"
+fi
 
-if [ -z "$PLUGIN" ]; then
+if [ -z "$DEVLAKE_PLUGINS" ]; then
     echo "Building all plugins"
     PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -empty)
 else
     echo "Building the following plugins: $PLUGIN"
     PLUGINS=
-    for p in $(echo "$PLUGIN" | tr "," "\n"); do
+    for p in $(echo "$DEVLAKE_PLUGINS" | tr "," "\n"); do
         PLUGINS="$PLUGINS $PLUGIN_SRC_DIR/$p"
     done
 fi
 
+
 rm -rf $PLUGIN_OUTPUT_DIR/*
 
 PIDS=""
 for PLUG in $PLUGINS; do
     NAME=$(basename $PLUG)
     echo "Building plugin $NAME to bin/plugins/$NAME/$NAME.so with args: $*"
-    go build -buildmode=plugin "$@" -o $PLUGIN_OUTPUT_DIR/$NAME/$NAME.so $PLUG/*.go &
+    go build -buildmode=plugin $EXTRA -o $PLUGIN_OUTPUT_DIR/$NAME/$NAME.so $PLUG/*.go &
     PIDS="$PIDS $!"
     # avoid too many processes causing signal killed
     COUNT=$(echo "$PIDS" | wc -w)
diff --git a/backend/scripts/build-python.sh b/backend/scripts/build-python.sh
new file mode 100755
index 000000000..7a065d748
--- /dev/null
+++ b/backend/scripts/build-python.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+ROOT_DIR=$(dirname $(dirname "$0"))
+find $ROOT_DIR/python/ -name "*.sh" | xargs chmod +x
+$ROOT_DIR/python/build.sh
+$ROOT_DIR/build.sh $ROOT_DIR/python/test
\ No newline at end of file
diff --git a/backend/scripts/build-server.sh b/backend/scripts/build-server.sh
new file mode 100755
index 000000000..edb7e3035
--- /dev/null
+++ b/backend/scripts/build-server.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+ROOT_DIR=$(dirname $(dirname "$0"))
+VERSION=${VERSION:-$(git describe --tags --always --dirty)}
+EXTRA=""
+
+if [ -n "$DEVLAKE_DEBUG" ]; then
+    EXTRA="-gcflags='all=-N -l'"
+fi
+
+go build $EXTRA -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$VERSION'" -o $ROOT_DIR/bin/lake $ROOT_DIR/server/
\ No newline at end of file
diff --git a/backend/scripts/e2e-test-go-plugins.sh b/backend/scripts/e2e-test-go-plugins.sh
new file mode 100644
index 000000000..3ace11ffa
--- /dev/null
+++ b/backend/scripts/e2e-test-go-plugins.sh
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+ROOT_DIR=$(dirname $(dirname "$0"))
+EXTRA=""
+
+if [ -n "$DEVLAKE_DEBUG" ]; then
+    EXTRA="-gcflags='all=-N -l'"
+fi
+
+export TZ=UTC
+export ENV_PATH=${ENV_PATH:-$ROOT_DIR/.env}
+
+for m in $(go list $ROOT_DIR/plugins/... | egrep 'e2e'); do
+  echo start running e2e test for plugin $m;
+  go test -timeout 300s $EXTRA -v $m
+done
\ No newline at end of file
diff --git a/backend/scripts/e2e-test.sh b/backend/scripts/e2e-test.sh
new file mode 100644
index 000000000..3edb8f79d
--- /dev/null
+++ b/backend/scripts/e2e-test.sh
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+ROOT_DIR=$(dirname $(dirname "$0"))
+EXTRA=""
+
+if [ -n "$DEVLAKE_DEBUG" ]; then
+    EXTRA="-gcflags='all=-N -l'"
+fi
+
+export TZ=UTC
+export ENV_PATH=${ENV_PATH:-$ROOT_DIR/.env}
+
+go run $ROOT_DIR/test/init.go
+for m in $(go list $ROOT_DIR/test/e2e/... | grep -v manual); do \
+  echo start running e2e test on $m
+  go test -failfast -timeout 120s -v $m
+done
\ No newline at end of file
diff --git a/backend/scripts/jira-api-benchmark.sh b/backend/scripts/jira-api-benchmark.sh
old mode 100644
new mode 100755
diff --git a/backend/scripts/unit-test-go.sh b/backend/scripts/unit-test-go.sh
new file mode 100755
index 000000000..eb3f63d00
--- /dev/null
+++ b/backend/scripts/unit-test-go.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+ROOT_DIR=$(dirname $(dirname "$0"))
+for m in $(go list $ROOT_DIR/... | egrep -v 'test|models|e2e'); do
+  echo start unit testing on $m
+  go test -timeout 60s -v $m
+done
\ No newline at end of file