You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@devlake.apache.org by he...@apache.org on 2023/07/06 22:46:18 UTC
[incubator-devlake] branch main updated: feat: make lake run with non-root users (#5583)
This is an automated email from the ASF dual-hosted git repository.
hez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 6a5fbf7a0 feat: make lake run with non-root users (#5583)
6a5fbf7a0 is described below
commit 6a5fbf7a08d0a9fffbae744ec7be0e7b094f90fc
Author: ZhangNing10 <13...@users.noreply.github.com>
AuthorDate: Fri Jul 7 06:46:13 2023 +0800
feat: make lake run with non-root users (#5583)
* feat: make devlake run in nonroot user
* feat: make devlake run as nonroot user and specify the user id
* feat: make lake run with non-root user devlake whose home dir is /app
* fix: adjust path
* feat: support for debug builds and plugin-whitelisting in Dockerfile for easier troubleshooting
---------
Co-authored-by: Keon Amini <ke...@merico.dev>
---
Makefile | 3 ---
backend/Dockerfile | 39 +++++++++++++++++++++++++-------------
backend/Makefile | 37 ++++++++++++++++++++++++++++--------
backend/impls/logruslog/stream.go | 2 +-
backend/scripts/compile-plugins.sh | 5 ++++-
5 files changed, 60 insertions(+), 26 deletions(-)
diff --git a/Makefile b/Makefile
index 1ad172eb8..8d70432d4 100644
--- a/Makefile
+++ b/Makefile
@@ -71,9 +71,6 @@ swag:
build-plugin:
make build-plugin -C backend
-build-plugin-debug:
- make build-plugin-debug -C backend
-
build-worker:
make build-worker -C backend
diff --git a/backend/Dockerfile b/backend/Dockerfile
index eef532cdf..f8532ba24 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -60,6 +60,7 @@ COPY --from=debian-arm64 /usr/include /rootfs-arm64/usr/include
COPY --from=debian-arm64 /usr/lib/aarch64-linux-gnu /rootfs-arm64/usr/lib/aarch64-linux-gnu
COPY --from=debian-arm64 /lib/aarch64-linux-gnu /rootfs-arm64/lib/aarch64-linux-gnu
+
RUN for arch in aarch64 x86_64 ; do \
mkdir -p /tmp/build/${arch} && cd /tmp/build/${arch} && \
wget https://github.com/libgit2/libgit2/archive/refs/tags/v1.3.2.tar.gz -O - | tar -xz && \
@@ -87,6 +88,8 @@ ENV GOBIN=/app/bin
ARG TARGETPLATFORM
ARG TAG=
ARG SHA=
+ARG GO_PLUGINS=
+ARG DEBUG=
RUN --mount=type=cache,target=/root/.cache/go-build \
if [ "$TARGETPLATFORM" = "linux/arm64" ] ; then \
@@ -100,7 +103,11 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
fi && \
export PKG_CONFIG_PATH=/usr/local/deps/target/lib/pkgconfig && \
export CGO_ENABLED=1 &&\
- make all
+ PLUGIN="$GO_PLUGINS" make all
+
+RUN if [ "$DEBUG" = "true" ]; then \
+ go install github.com/go-delve/delve/cmd/dlv@latest; \
+ fi;
# remove symlink in lib, we will recreate in final image
RUN cd /usr/local/deps/target/lib && \
@@ -113,10 +120,8 @@ RUN cd /usr/local/deps/target/lib && \
FROM python:3.9-slim-bullseye as base
-ENV PYTHONUNBUFFERED=1
-
RUN apt-get update && \
- apt-get install -y python3-dev python3-pip tar curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev && \
+ apt-get install -y python3-dev python3-pip tar curl libssh2-1 zlib1g libffi-dev default-libmysqlclient-dev libpq-dev tini && \
apt-get clean && \
rm -fr /usr/share/doc/* \
/usr/share/info/* \
@@ -132,26 +137,36 @@ RUN apt-get update && \
EXPOSE 8080
+RUN useradd -ms /bin/bash -d /app devlake -u 1010
+USER devlake
+ENV PYTHONUNBUFFERED=1
+
WORKDIR /app
+
+RUN mkdir logs
+VOLUME /app/logs
+
# Setup Python
COPY python/ /app/python/
RUN python3 -m pip install --no-cache --upgrade pip setuptools && \
- python3 -m pip install --no-cache -r python/requirements.txt && \
- rm -fr /usr/share/python-wheels/*
+ python3 -m pip install --no-cache -r python/requirements.txt
+
# Setup Python Poetry package manager
RUN curl -sSL https://install.python-poetry.org | python3 -
-RUN ln -sf /root/.local/bin/poetry /usr/local/bin
-# Build Python plugins
-RUN find /app/python/ -name "*.sh" | xargs -I{} chmod +x {}
-RUN /app/python/build.sh
+ENV PATH="$PATH:/app/.local/bin"
+# Build Python plugins, make sure the scripts has execute permission
+# RUN find /app/python/ -name "*.sh" | xargs -I{} chmod +x {}
+RUN /app/python/build.sh
FROM base as devlake-base
+ARG DEBUG=
# libraries
ENV LD_LIBRARY_PATH=/app/libs
RUN mkdir -p /app/libs
COPY --from=build /usr/local/deps/target/lib/*.so* /app/libs
+COPY --from=build /go/bin /usr/bin
RUN ldconfig -vn /app/libs
# apps
@@ -159,11 +174,9 @@ COPY --from=build /app/bin /app/bin
COPY --from=build /app/resources /app/resources
ENV PATH="/app/bin:${PATH}"
+ENV DEBUG="$DEBUG"
#add tini, prevent zombie process
-RUN apt-get update && \
- apt-get install -y tini
-
ENTRYPOINT ["/usr/bin/tini", "--"]
CMD ["lake"]
diff --git a/backend/Makefile b/backend/Makefile
index 38a18751a..c55a019f9 100644
--- a/backend/Makefile
+++ b/backend/Makefile
@@ -35,20 +35,41 @@ python-dep:
dep: go-dep python-dep
swag:
- swag init --parseDependency --parseInternal -o ./server/api/docs -g ./server/api/api.go -g ./plugins/*/api/*.go
- @echo "visit the swagger document on http://localhost:8080/swagger/index.html"
+ if [ -z $(PLUGIN) ]; then \
+ swag init --parseDependency --parseInternal -o ./server/api/docs -g ./server/api/api.go -g ./plugins/*/api/*.go; \
+ elif [ $(PLUGIN) = "none" ]; then \
+ swag init --parseDependency --parseInternal -o ./server/api/docs -g ./server/api/api.go;\
+ else \
+ plugins="";\
+ for p in $$(echo $(PLUGIN) | tr "," "\n"); do \
+ plugins="$$plugins -g ./plugins/$$p/api/*.go"; \
+ done;\
+ swag init --parseDependency --parseInternal -o ./server/api/docs -g ./server/api/api.go "$$plugins"; \
+ fi;\
+ echo "visit the swagger document on http://localhost:8080/swagger/index.html";
build-plugin:
- @sh scripts/compile-plugins.sh
-
-build-plugin-debug:
- @sh scripts/compile-plugins.sh -gcflags='all=-N -l'
+ if [ "$(PLUGIN)" = "none" ]; then \
+ echo "Building plugins will be skipped"; \
+ elif [ "$(DEBUG)" = "true" ]; then \
+ PLUGIN=$(PLUGIN) sh scripts/compile-plugins.sh -gcflags='all=-N -l'; \
+ else \
+ PLUGIN=$(PLUGIN) sh scripts/compile-plugins.sh; \
+ fi
build-worker:
- go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake-worker ./worker/
+ if [ "$(DEBUG)" = "true" ]; then \
+ go build -gcflags='all=-N -l' -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake-worker ./worker/; \
+ else \
+ go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake-worker ./worker/; \
+ fi
build-server: swag
- go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/
+ if [ "$(DEBUG)" = "true" ]; then \
+ go build -gcflags='all=-N -l' -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/; \
+ else \
+ go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/; \
+ fi
build-python: #don't mix this with the other build commands
find ./python/ -name "*.sh" | xargs chmod +x &&\
diff --git a/backend/impls/logruslog/stream.go b/backend/impls/logruslog/stream.go
index 69f529531..d5429e19b 100644
--- a/backend/impls/logruslog/stream.go
+++ b/backend/impls/logruslog/stream.go
@@ -28,7 +28,7 @@ func GetFileStream(path string) (io.Writer, errors.Error) {
if path == "" {
return os.Stdout, nil
}
- err := os.MkdirAll(filepath.Dir(path), os.ModePerm)
+ err := os.MkdirAll(filepath.Dir(path), 0777)
if err != nil {
return nil, errors.Convert(err)
}
diff --git a/backend/scripts/compile-plugins.sh b/backend/scripts/compile-plugins.sh
index 9b4d7fd4f..4ee940d3f 100644
--- a/backend/scripts/compile-plugins.sh
+++ b/backend/scripts/compile-plugins.sh
@@ -38,9 +38,12 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
PLUGIN_SRC_DIR=$SCRIPT_DIR/../plugins
PLUGIN_OUTPUT_DIR=$SCRIPT_DIR/../bin/plugins
+
if [ -z "$PLUGIN" ]; then
+ echo "Building all plugins"
PLUGINS=$(find $PLUGIN_SRC_DIR/* -maxdepth 0 -type d -not -name core -not -name helper -not -name logs -not -empty)
else
+ echo "Building the following plugins: $PLUGIN"
PLUGINS=
for p in $(echo "$PLUGIN" | tr "," "\n"); do
PLUGINS="$PLUGINS $PLUGIN_SRC_DIR/$p"
@@ -52,7 +55,7 @@ rm -rf $PLUGIN_OUTPUT_DIR/*
PIDS=""
for PLUG in $PLUGINS; do
NAME=$(basename $PLUG)
- echo "Building plugin $NAME to bin/plugins/$NAME/$NAME.so"
+ echo "Building plugin $NAME to bin/plugins/$NAME/$NAME.so with args: $*"
go build -buildmode=plugin "$@" -o $PLUGIN_OUTPUT_DIR/$NAME/$NAME.so $PLUG/*.go &
PIDS="$PIDS $!"
# avoid too many processes causing signal killed