You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2021/01/21 20:51:40 UTC
[couchdb] branch main updated: Simplify and speedup dev node
startup (main branch) (#3338)
This is an automated email from the ASF dual-hosted git repository.
kocolosk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new b033ef5 Simplify and speedup dev node startup (main branch) (#3338)
b033ef5 is described below
commit b033ef59e7b231d0620e266ee376a430101e54f6
Author: Adam Kocoloski <ko...@apache.org>
AuthorDate: Thu Jan 21 15:51:30 2021 -0500
Simplify and speedup dev node startup (main branch) (#3338)
* Simplify and speedup dev node startup
This patch introduces an escript that generates an Erlang .boot script
to start CouchDB using the in-place .beam files produced by the compile
phase of the build. This allows us to radically simplify the boot
process as Erlang computes the optimal order for loading the necessary
modules.
In addition to the simplification this approach offers a significant
speedup when working inside a container environment. In my test with
the stock .devcontainer it reduces startup time from about 75 seconds
down to under 5 seconds.
* Rename boot_node to monitor_parent
* Add formatting suggestions from python-black
Co-authored-by: Paul J. Davis <pa...@gmail.com>
---
.gitignore | 3 +-
Makefile | 2 +-
dev/boot_node.erl | 148 -------------------------------------------------
dev/make_boot_script | 9 +++
dev/monitor_parent.erl | 43 ++++++++++++++
dev/run | 19 +++++--
6 files changed, 69 insertions(+), 155 deletions(-)
diff --git a/.gitignore b/.gitignore
index de31279..7192941 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,7 +23,8 @@ bin/
config.erl
*.tar.gz
*.tar.bz2
-dev/boot_node.beam
+dev/*.beam
+dev/devnode.*
dev/lib/
dev/logs/
ebin/
diff --git a/Makefile b/Makefile
index e4c153e..b02f180 100644
--- a/Makefile
+++ b/Makefile
@@ -449,7 +449,7 @@ clean:
@rm -rf src/mango/.venv
@rm -f src/couch/priv/couchspawnkillable
@rm -f src/couch/priv/couch_js/config.h
- @rm -f dev/boot_node.beam dev/pbkdf2.pyc log/crash.log
+ @rm -f dev/*.beam dev/devnode.* dev/pbkdf2.pyc log/crash.log
.PHONY: distclean
diff --git a/dev/boot_node.erl b/dev/boot_node.erl
deleted file mode 100644
index 922a5cc..0000000
--- a/dev/boot_node.erl
+++ /dev/null
@@ -1,148 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(boot_node).
-
--export([start/0]).
-
-
-start() ->
- monitor_parent(),
- Apps = load_apps(),
- Deps = load_deps(Apps),
- start_all_apps(Deps).
-
-
-monitor_parent() ->
- {ok, [[PPid]]} = init:get_argument(parent_pid),
- spawn(fun() -> monitor_parent(PPid) end).
-
-
-monitor_parent(PPid) ->
- timer:sleep(1000),
- case os:type() of
- {unix, _} ->
- case os:cmd("kill -0 " ++ PPid) of
- "" ->
- monitor_parent(PPid);
- _Else ->
- % Assume _Else is a no such process error
- init:stop()
- end;
- {win32, _} ->
- Fmt = "tasklist /fi \"PID eq ~s\" /fo csv /nh",
- Retval = os:cmd(io_lib:format(Fmt, [PPid])),
- case re:run(Retval, "^\"python.exe\",*") of
- {match, _} ->
- monitor_parent(PPid);
- nomatch ->
- init:stop()
- end
- end.
-
-
-load_apps() ->
- {ok, [[Config]]} = init:get_argument(reltool_config),
- {ok, Terms} = file:consult(Config),
- load_apps(Terms).
-
-
-load_apps([]) ->
- erlang:error(failed_to_load_apps);
-load_apps([{sys, Terms} | _]) ->
- load_apps(Terms);
-load_apps([{rel, "couchdb", _Vsn, Apps} | _]) ->
- Apps;
-load_apps([_ | Rest]) ->
- load_apps(Rest).
-
-
-load_deps(Apps) ->
- load_deps(Apps, dict:new()).
-
-
-load_deps([], Deps) ->
- Deps;
-load_deps([App | Rest], Deps) ->
- load_app(App),
- case application:get_key(App, applications) of
- {ok, AppDeps0} ->
- NewDeps = dict:store(App, AppDeps0, Deps),
- Filter = fun(A) -> not dict:is_key(A, Deps) end,
- AppDeps = lists:filter(Filter, AppDeps0),
- load_deps(AppDeps ++ Rest, NewDeps);
- _ ->
- NewDeps = dict:store(App, [], Deps),
- load_deps(Rest, NewDeps)
- end.
-
-
-load_app(App) ->
- case application:load(App) of
- ok ->
- case application:get_key(App, modules) of
- {ok, Modules} ->
- lists:foreach(fun(Mod) ->
- case load_app_module(Mod) of
- ok -> ok;
- E -> io:format("~p = load_app_module(~p)~n", [E, Mod])
- end
- end, Modules);
- undefined ->
- ok
- end;
- {error, {already_loaded, App}} ->
- ok;
- Error ->
- Error
- end.
-
-
-load_app_module(Mod) ->
- case code:is_loaded(Mod) of
- {file, _} ->
- ok;
- _ ->
- case code:load_file(Mod) of
- {module, Mod} ->
- ok;
- Error ->
- Error
- end
- end.
-
-
-start_all_apps(Deps) ->
- lists:foldl(fun(App, Started) ->
- start_app(App, Deps, Started)
- end, [], dict:fetch_keys(Deps)).
-
-
-start_app(App, Deps, Started) ->
- case lists:member(App, Started) of
- true ->
- Started;
- false ->
- AppDeps = dict:fetch(App, Deps),
- NowStarted = lists:foldl(fun(Dep, Acc) ->
- start_app(Dep, Deps, Acc)
- end, Started, AppDeps),
- case application:start(App) of
- ok ->
- [App | NowStarted];
- {error, {already_started,App}} ->
- % Kernel causes this
- [App | NowStarted];
- Else ->
- erlang:error(Else)
- end
- end.
diff --git a/dev/make_boot_script b/dev/make_boot_script
new file mode 100755
index 0000000..549dd9a
--- /dev/null
+++ b/dev/make_boot_script
@@ -0,0 +1,9 @@
+#!/usr/bin/env escript
+
+main(_) ->
+ {ok, Server} = reltool:start_server([
+ {config, "../rel/reltool.config"}
+ ]),
+ {ok, Release} = reltool:get_rel(Server, "couchdb"),
+ ok = file:write_file("devnode.rel", io_lib:format("~p.~n", [Release])),
+ ok = systools:make_script("devnode", [local]).
diff --git a/dev/monitor_parent.erl b/dev/monitor_parent.erl
new file mode 100644
index 0000000..382f37e
--- /dev/null
+++ b/dev/monitor_parent.erl
@@ -0,0 +1,43 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(monitor_parent).
+
+-export([start/0]).
+
+
+start() ->
+ {ok, [[PPid]]} = init:get_argument(parent_pid),
+ spawn(fun() -> monitor_parent(PPid) end).
+
+
+monitor_parent(PPid) ->
+ timer:sleep(1000),
+ case os:type() of
+ {unix, _} ->
+ case os:cmd("kill -0 " ++ PPid) of
+ "" ->
+ monitor_parent(PPid);
+ _Else ->
+ % Assume _Else is a no such process error
+ init:stop()
+ end;
+ {win32, _} ->
+ Fmt = "tasklist /fi \"PID eq ~s\" /fo csv /nh",
+ Retval = os:cmd(io_lib:format(Fmt, [PPid])),
+ case re:run(Retval, "^\"python.exe\",*") of
+ {match, _} ->
+ monitor_parent(PPid);
+ nomatch ->
+ init:stop()
+ end
+ end.
diff --git a/dev/run b/dev/run
index d620e21..7af718a 100755
--- a/dev/run
+++ b/dev/run
@@ -101,6 +101,7 @@ def setup():
setup_logging(ctx)
setup_dirs(ctx)
check_beams(ctx)
+ check_boot_script(ctx)
setup_configs(ctx)
return ctx
@@ -189,7 +190,7 @@ def get_args_parser():
"--erlang-config",
dest="erlang_config",
default="rel/files/sys.config",
- help="Specify an alternative Erlang application configuration"
+ help="Specify an alternative Erlang application configuration",
)
parser.add_option(
"--degrade-cluster",
@@ -275,6 +276,14 @@ def check_beams(ctx):
sp.check_call(["erlc", "-o", ctx["devdir"] + os.sep, fname])
+@log("Ensure Erlang boot script exists")
+def check_boot_script(ctx):
+ if not os.path.exists(os.path.join(ctx["devdir"], "devnode.boot")):
+ env = os.environ.copy()
+ env["ERL_LIBS"] = os.path.join(ctx["rootdir"], "src")
+ sp.check_call(["escript", "make_boot_script"], env=env, cwd=ctx["devdir"])
+
+
@log("Prepare configuration files")
def setup_configs(ctx):
for idx, node in enumerate(ctx["nodes"]):
@@ -599,10 +608,9 @@ def set_boot_env(ctx):
@log("Start node {node}")
def boot_node(ctx, node):
- erl_libs = os.path.join(ctx["rootdir"], "src")
set_boot_env(ctx)
env = os.environ.copy()
- env["ERL_LIBS"] = os.pathsep.join([erl_libs])
+ env["ERL_LIBS"] = os.path.join(ctx["rootdir"], "src")
node_etcdir = os.path.join(ctx["devdir"], "lib", node, "etc")
reldir = os.path.join(ctx["rootdir"], "rel")
@@ -621,11 +629,12 @@ def boot_node(ctx, node):
os.path.join(reldir, "reltool.config"),
"-parent_pid",
str(os.getpid()),
+ "-boot",
+ os.path.join(ctx["devdir"], "devnode"),
"-pa",
ctx["devdir"],
+ "-s monitor_parent",
]
- cmd += [p[:-1] for p in glob.glob(erl_libs + "/*/")]
- cmd += ["-s", "boot_node"]
if ctx["reset_logs"]:
mode = "wb"
else: