You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by tz...@apache.org on 2022/12/16 02:09:27 UTC

[apisix] branch master updated: feat: add inspect plugin (#8400)

This is an automated email from the ASF dual-hosted git repository.

tzssangglass pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git


The following commit(s) were added to refs/heads/master by this push:
     new 8486800a7 feat: add inspect plugin (#8400)
8486800a7 is described below

commit 8486800a7cb7bffa74e5a59876acdd61191d3162
Author: jinhua luo <ho...@163.com>
AuthorDate: Fri Dec 16 10:09:22 2022 +0800

    feat: add inspect plugin (#8400)
---
 Makefile                              |   3 +
 apisix/inspect/dbg.lua                | 151 ++++++++++
 apisix/inspect/init.lua               | 128 +++++++++
 apisix/plugins/inspect.lua            |  61 +++++
 conf/config-default.yaml              |   4 +
 docs/assets/images/plugin/inspect.png | Bin 0 -> 31490 bytes
 docs/en/latest/config.json            |   3 +-
 docs/en/latest/plugins/inspect.md     | 171 ++++++++++++
 t/admin/plugins.t                     |   1 +
 t/lib/test_inspect.lua                |  62 +++++
 t/plugin/inspect.t                    | 499 ++++++++++++++++++++++++++++++++++
 11 files changed, 1082 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index f5908f71f..385e8ed46 100644
--- a/Makefile
+++ b/Makefile
@@ -301,6 +301,9 @@ install: runtime
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/include/apisix/model
 	$(ENV_INSTALL) apisix/include/apisix/model/*.proto $(ENV_INST_LUADIR)/apisix/include/apisix/model/
 
+	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/inspect
+	$(ENV_INSTALL) apisix/inspect/*.lua $(ENV_INST_LUADIR)/apisix/inspect/
+
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins
 	$(ENV_INSTALL) apisix/plugins/*.lua $(ENV_INST_LUADIR)/apisix/plugins/
 
diff --git a/apisix/inspect/dbg.lua b/apisix/inspect/dbg.lua
new file mode 100644
index 000000000..a8a619a26
--- /dev/null
+++ b/apisix/inspect/dbg.lua
@@ -0,0 +1,151 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local string_format = string.format
+local debug = debug
+local ipairs = ipairs
+local pcall = pcall
+local table_insert = table.insert
+local jit = jit
+
+local _M = {}
+
+local hooks = {}
+
+function _M.getname(n)
+    if n.what == "C" then
+        return n.name
+    end
+    local lc = string_format("%s:%d", n.short_src, n.currentline)
+    if n.what ~= "main" and n.namewhat ~= "" then
+        return string_format("%s (%s)", lc, n.name)
+    else
+        return lc
+    end
+end
+
+local function hook(_, arg)
+    local level = 2
+    local finfo = debug.getinfo(level, "nSlf")
+    local key = finfo.source .. "#" .. arg
+
+    local hooks2 = {}
+    for _, hook in ipairs(hooks) do
+        if key:sub(-#hook.key) == hook.key then
+            local filter_func = hook.filter_func
+            local info = {finfo = finfo, uv = {}, vals = {}}
+
+            -- upvalues
+            local i = 1
+            while true do
+                local name, value = debug.getupvalue(finfo.func, i)
+                if name == nil then break end
+                if name:sub(1, 1) ~= "(" then
+                    info.uv[name] = value
+                end
+                i = i + 1
+            end
+
+            -- local values
+            local i = 1
+            while true do
+                local name, value = debug.getlocal(level, i)
+                if not name then break end
+                if name:sub(1, 1) ~= "(" then
+                    info.vals[name] = value
+                end
+                i = i + 1
+            end
+
+            local r1, r2_or_err = pcall(filter_func, info)
+            if not r1 then
+                core.log.error("inspect: pcall filter_func:", r2_or_err)
+            elseif r2_or_err == false then
+                -- if filter_func returns false, keep the hook
+                table_insert(hooks2, hook)
+            end
+        else
+            -- key not match, keep the hook
+            table_insert(hooks2, hook)
+        end
+    end
+
+    -- disable debug mode if all hooks done
+    if #hooks2 ~= #hooks then
+        hooks = hooks2
+        if #hooks == 0 then
+            debug.sethook()
+        end
+    end
+end
+
+function _M.set_hook(file, line, func, filter_func)
+    if file == nil then
+        file = "=stdin"
+    end
+
+    local key = file .. "#" .. line
+    table_insert(hooks, {key = key, filter_func = filter_func})
+
+    if jit then
+        jit.flush(func)
+        jit.off()
+    end
+
+    debug.sethook(hook, "l")
+end
+
+function _M.unset_hook(file, line)
+    if file == nil then
+        file = "=stdin"
+    end
+
+    local hooks2 = {}
+
+    local key = file .. "#" .. line
+    for i, hook in ipairs(hooks) do
+        if hook.key ~= key then
+            table_insert(hooks2, hook)
+        end
+    end
+
+    if #hooks2 ~= #hooks then
+        hooks = hooks2
+        if #hooks == 0 then
+            debug.sethook()
+            if jit then
+                jit.on()
+            end
+        end
+    end
+end
+
+function _M.unset_all()
+    if #hooks > 0 then
+        hooks = {}
+        debug.sethook()
+        if jit then
+            jit.on()
+        end
+    end
+end
+
+function _M.hooks()
+    return hooks
+end
+
+return _M
diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua
new file mode 100644
index 000000000..a33c30ec3
--- /dev/null
+++ b/apisix/inspect/init.lua
@@ -0,0 +1,128 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local dbg = require("apisix.inspect.dbg")
+local lfs = require("lfs")
+local pl_path = require("pl.path")
+local io = io
+local table_insert = table.insert
+local pcall = pcall
+local ipairs = ipairs
+local os = os
+local ngx = ngx
+local loadstring = loadstring
+local format = string.format
+
+local _M = {}
+
+local last_modified = 0
+
+local stop = false
+
+local running = false
+
+local last_report_time = 0
+
+local REPORT_INTERVAL = 30 -- secs
+
+local function run_lua_file(file)
+    local f, err = io.open(file, "rb")
+    if not f then
+        return false, err
+    end
+    local code, err = f:read("*all")
+    f:close()
+    if code == nil then
+        return false, format("cannot read hooks file: %s", err)
+    end
+    local func, err = loadstring(code)
+    if not func then
+        return false, err
+    end
+    func()
+    return true
+end
+
+local function setup_hooks(file)
+    if pl_path.exists(file) then
+        dbg.unset_all()
+        local _, err = pcall(run_lua_file, file)
+        local hooks = {}
+        for _, hook in ipairs(dbg.hooks()) do
+            table_insert(hooks, hook.key)
+        end
+        core.log.info("set hooks: err: ", err, ", hooks: ", core.json.delay_encode(hooks))
+    end
+end
+
+local function reload_hooks(premature, delay, file)
+    if premature or stop then
+        stop = false
+        running = false
+        return
+    end
+
+    local time, err = lfs.attributes(file, 'modification')
+    if err then
+        if last_modified ~= 0 then
+            core.log.info(err, ", disable all hooks")
+            dbg.unset_all()
+            last_modified = 0
+        end
+    elseif time ~= last_modified then
+        setup_hooks(file)
+        last_modified = time
+    else
+        local ts = os.time()
+        if ts - last_report_time >= REPORT_INTERVAL then
+            local hooks = {}
+            for _, hook in ipairs(dbg.hooks()) do
+                table_insert(hooks, hook.key)
+            end
+            core.log.info("alive hooks: ", core.json.encode(hooks))
+            last_report_time = ts
+        end
+    end
+
+    local ok, err = ngx.timer.at(delay, reload_hooks, delay, file)
+    if not ok then
+        core.log.error("failed to create the timer: ", err)
+        running = false
+    end
+end
+
+function _M.init(delay, file)
+    if not running then
+        file = file or "/var/run/apisix_inspect_hooks.lua"
+        delay = delay or 3
+
+        setup_hooks(file)
+
+        local ok, err = ngx.timer.at(delay, reload_hooks, delay, file)
+        if not ok then
+            core.log.error("failed to create the timer: ", err)
+            return
+        end
+        running = true
+    end
+end
+
+function _M.destroy()
+    stop = true
+end
+
+return _M
diff --git a/apisix/plugins/inspect.lua b/apisix/plugins/inspect.lua
new file mode 100644
index 000000000..19f50c79e
--- /dev/null
+++ b/apisix/plugins/inspect.lua
@@ -0,0 +1,61 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local plugin = require("apisix.plugin")
+local inspect = require("apisix.inspect")
+
+
+local plugin_name = "inspect"
+
+
+local schema = {
+    type = "object",
+    properties = {},
+}
+
+
+local _M = {
+    version = 0.1,
+    priority = 200,
+    name = plugin_name,
+    schema = schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+    return core.schema.check(schema, conf)
+end
+
+
+function _M.init()
+    local attr = plugin.plugin_attr(plugin_name)
+    local delay
+    local hooks_file
+    if attr then
+        delay = attr.delay
+        hooks_file = attr.hooks_file
+    end
+    core.log.info("delay=", delay, ", hooks_file=", hooks_file)
+    return inspect.init(delay, hooks_file)
+end
+
+
+function _M.destroy()
+    return inspect.destroy()
+end
+
+return _M
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 59a6de6b0..33af44564 100755
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -491,6 +491,7 @@ plugins:                          # plugin list (sorted by priority)
   - file-logger                    # priority: 399
   - clickhouse-logger              # priority: 398
   - tencent-cloud-cls              # priority: 397
+  - inspect                        # priority: 200
   #- log-rotate                    # priority: 100
   # <- recommend to use priority (0, 100) for your custom plugins
   - example-plugin                 # priority: 0
@@ -586,6 +587,9 @@ plugin_attr:
       send: 60s
 #  redirect:
 #    https_port: 8443   # the default port for use by HTTP redirects to HTTPS
+  inspect:
+    delay: 3            # in seconds
+    hooks_file: "/usr/local/apisix/plugin_inspect_hooks.lua"
 
 deployment:
   role: traditional
diff --git a/docs/assets/images/plugin/inspect.png b/docs/assets/images/plugin/inspect.png
new file mode 100644
index 000000000..efe82eed8
Binary files /dev/null and b/docs/assets/images/plugin/inspect.png differ
diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json
index 73836ab41..8e3dbb3d4 100644
--- a/docs/en/latest/config.json
+++ b/docs/en/latest/config.json
@@ -60,7 +60,8 @@
             "plugins/server-info",
             "plugins/ext-plugin-pre-req",
             "plugins/ext-plugin-post-req",
-            "plugins/ext-plugin-post-resp"
+            "plugins/ext-plugin-post-resp",
+            "plugins/inspect"
           ]
         },
         {
diff --git a/docs/en/latest/plugins/inspect.md b/docs/en/latest/plugins/inspect.md
new file mode 100644
index 000000000..fad20a1bb
--- /dev/null
+++ b/docs/en/latest/plugins/inspect.md
@@ -0,0 +1,171 @@
+---
+title: inspect
+keywords:
+  - APISIX
+  - Plugin
+  - Inspect
+  - Dynamic Lua Debugging
+description: This document contains information about the Apache APISIX inspect Plugin.
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## Description
+
+It's useful to set arbitrary breakpoint in any Lua file to inspect the context information,
+e.g. print local variables if some condition satisfied.
+
+In this way, you don't need to modify the source code of your project, and just get diagnose information
+on demand, i.e. dynamic logging.
+
+This plugin supports setting breakpoints within both interpretd function and jit compiled function.
+The breakpoint could be at any position within the function. The function could be global/local/module/ananymous.
+
+## Features
+
+* Set breakpoint at any position
+* Dynamic breakpoint
+* customized breakpoint handler
+* You could define one-shot breakpoint
+* Work for jit compiled function
+* If function reference specified, then performance impact is only bound to that function (JIT compiled code will not trigger debug hook, so they would run fast even if hook is enabled)
+* If all breakpoints deleted, jit could recover
+
+## Operation Graph
+
+![Operation Graph](https://raw.githubusercontent.com/apache/apisix/master/docs/assets/images/plugin/inspect.png)
+
+## API to define hook in hooks file
+
+### require("resty.inspect.dbg").set_hook(file, line, func, filter_func)
+
+The breakpoint is specified by `file` (full qualified or short file name) and the `line` number.
+
+The `func` specified the scope (which function or global) of jit cache to flush:
+
+* If the breakpoint is related to a module function or
+global function, you should set it that function reference, then only the jit cache of that function would
+be flushed, and it would not affect other caches to avoid slowing down other parts of the program.
+
+* If the breakpointis related to local function or anonymous function,
+then you have to set it to `nil` (because no way to get function reference), which would flush the whole jit cache of Lua vm.
+
+You attach a `filter_func` function of the breakpoint, the function takes the `info` as argument and returns
+true of false to determine whether the breakpoint would be removed. You could setup one-shot breakpoint
+at ease.
+
+The `info` is a hash table which contains below keys:
+
+* `finfo`: `debug.getinfo(level, "nSlf")`
+* `uv`: upvalues hash table
+* `vals`: local variables hash table
+
+## Attributes
+
+| Name               | Type    | Required | Default | Description                                                                                    |
+|--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------|
+| delay           | integer | False     | 3 | Time in seconds specifying how often to check the hooks file.                                       |
+| hooks_file           | string | False     | "/usr/local/apisix/plugin_inspect_hooks.lua"  | Lua file to define hooks, which could be a link file. Ensure only administrator could write this file, otherwise it may be a security risk. |
+
+## Enabling the Plugin
+
+Plugin is enabled by default (`conf/config-default.yaml`):
+
+```yaml title="conf/config-default.yaml"
+plugins:
+    - inspect
+
+plugin_attr:
+  inspect:
+    delay: 3
+    hooks_file: "/usr/local/apisix/plugin_inspect_hooks.lua"
+```
+
+## Example usage
+
+```bash
+# create test route
+curl http://127.0.0.1:9180/apisix/admin/routes/test_limit_req -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "methods": ["GET"],
+    "uri": "/get",
+    "plugins": {
+        "limit-req": {
+            "rate": 100,
+            "burst": 0,
+            "rejected_code": 503,
+            "key_type": "var",
+            "key": "remote_addr"
+        }
+    },
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "httpbin.org": 1
+        }
+    }
+}'
+
+# create a hooks file to set a test breakpoint
+# Note that the breakpoint is associated with the line number,
+# so if the Lua code changes, you need to adjust the line number in the hooks file
+cat <<EOF >/usr/local/apisix/example_hooks.lua
+local dbg = require "resty.inspect.dbg"
+
+dbg.set_hook("limit-req.lua", 88, require("apisix.plugins.limit-req").access, function(info)
+    ngx.log(ngx.INFO, debug.traceback("foo traceback", 3))
+    ngx.log(ngx.INFO, dbg.getname(info.finfo))
+    ngx.log(ngx.INFO, "conf_key=", info.vals.conf_key)
+    return true
+end)
+
+--- more breakpoints could be defined via dbg.set_hook()
+--- ...
+EOF
+
+# enable the hooks file
+ln -sf /usr/local/apisix/example_hooks.lua /usr/local/apisix/plugin_inspect_hooks.lua
+
+# check errors.log to confirm the test breakpoint is enabled
+2022/09/01 00:55:38 [info] 2754534#2754534: *3700 [lua] init.lua:29: setup_hooks(): set hooks: err=nil, hooks=["limit-req.lua#88"], context: ngx.timer
+
+# access the test route
+curl -i http://127.0.0.1:9080/get
+
+# check errors.log to confirm the test breakpoint is triggered
+2022/09/01 00:55:52 [info] 2754534#2754534: *4070 [lua] resty_inspect_hooks.lua:4: foo traceback
+stack traceback:
+        /opt/lua-resty-inspect/lib/resty/inspect/dbg.lua:50: in function </opt/lua-resty-inspect/lib/resty/inspect/dbg.lua:17>
+        /opt/apisix.fork/apisix/plugins/limit-req.lua:88: in function 'phase_func'
+        /opt/apisix.fork/apisix/plugin.lua:900: in function 'run_plugin'
+        /opt/apisix.fork/apisix/init.lua:456: in function 'http_access_phase'
+        access_by_lua(nginx.conf:303):2: in main chunk, client: 127.0.0.1, server: _, request: "GET /get HTTP/1.1", host: "127.0.0.1:9080"
+2022/09/01 00:55:52 [info] 2754534#2754534: *4070 [lua] resty_inspect_hooks.lua:5: /opt/apisix.fork/apisix/plugins/limit-req.lua:88 (phase_func), client: 127.0.0.1, server: _, request: "GET /get HTTP/1.1", host: "127.0.0.1:9080"
+2022/09/01 00:55:52 [info] 2754534#2754534: *4070 [lua] resty_inspect_hooks.lua:6: conf_key=remote_addr, client: 127.0.0.1, server: _, request: "GET /get HTTP/1.1", host: "127.0.0.1:9080"
+```
+
+## Disable plugin
+
+To remove the `inspect` Plugin, you can remove it from your configuration file (`conf/config.yaml`):
+
+```yaml title="conf/config.yaml"
+plugins:
+    # - inspect
+```
diff --git a/t/admin/plugins.t b/t/admin/plugins.t
index 00a500e08..5edb3c427 100644
--- a/t/admin/plugins.t
+++ b/t/admin/plugins.t
@@ -123,6 +123,7 @@ udp-logger
 file-logger
 clickhouse-logger
 tencent-cloud-cls
+inspect
 example-plugin
 aws-lambda
 azure-functions
diff --git a/t/lib/test_inspect.lua b/t/lib/test_inspect.lua
new file mode 100644
index 000000000..62de59930
--- /dev/null
+++ b/t/lib/test_inspect.lua
@@ -0,0 +1,62 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--
+-- Don't edit existing code, because the hooks are identified by line number.
+-- Instead, append new code to this file.
+--
+local _M = {}
+
+function _M.run1()
+    local var1 = "hello"
+    local var2 = "world"
+    return var1 .. var2
+end
+
+local upvar1 = 2
+local upvar2 = "yes"
+function _M.run2()
+    return upvar1
+end
+
+function _M.run3()
+    return upvar1 .. upvar2
+end
+
+local str = string.rep("a", 8192) .. "llzz"
+
+local sk = require("socket")
+
+function _M.hot1()
+    local t1 = sk.gettime()
+    for i=1,100000 do
+        string.find(str, "ll", 1, true)
+    end
+    local t2 = sk.gettime()
+    return t2 - t1
+end
+
+function _M.hot2()
+    local t1 = sk.gettime()
+    for i=1,100000 do
+        string.find(str, "ll", 1, true)
+    end
+    local t2 = sk.gettime()
+    return t2 - t1
+end
+
+return _M
diff --git a/t/plugin/inspect.t b/t/plugin/inspect.t
new file mode 100644
index 000000000..e938431b4
--- /dev/null
+++ b/t/plugin/inspect.t
@@ -0,0 +1,499 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+log_level('warn');
+repeat_each(1);
+no_long_string();
+no_root_location();
+no_shuffle();
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!defined $block->request) {
+        $block->set_value("request", "GET /t");
+    }
+
+    my $user_yaml_config = <<_EOC_;
+plugin_attr:
+  inspect:
+    delay: 1
+    hooks_file: "/tmp/apisix_inspect_hooks.lua"
+_EOC_
+    $block->set_value("yaml_config", $user_yaml_config);
+
+    my $extra_init_worker_by_lua = $block->extra_init_worker_by_lua // "";
+    $extra_init_worker_by_lua .= <<_EOC_;
+local function gen_funcs_invoke(...)
+    local code = ""
+    for _, func in ipairs({...}) do
+        code = code .. "test." .. func .. "();"
+    end
+    return code
+end
+function set_test_route(...)
+    func = func or 'run1'
+    local t = require("lib.test_admin").test
+    local code = [[{
+        "methods": ["GET"],
+        "uri": "/inspect",
+        "plugins": {
+            "serverless-pre-function": {
+                "phase": "rewrite",
+                "functions" : ["return function() local test = require(\\"lib.test_inspect\\");]]
+                .. gen_funcs_invoke(...)
+                .. [[ngx.say(\\"ok\\"); end"]
+            }
+        },
+        "upstream": {
+            "type": "roundrobin",
+            "nodes": {
+                "127.0.0.1:1980": 1
+            }
+        }
+    }]]
+    return t('/apisix/admin/routes/inspect', ngx.HTTP_PUT, code)
+end
+
+function do_request()
+    local http = require "resty.http"
+    local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/inspect"
+
+    local httpc = http.new()
+    local res = httpc:request_uri(uri, {method = "GET"})
+    assert(res.body == "ok\\n")
+end
+
+function write_hooks(code, file)
+    local file = io.open(file or "/tmp/apisix_inspect_hooks.lua", "w")
+    file:write(code)
+    file:close()
+end
+_EOC_
+    $block->set_value("extra_init_worker_by_lua", $extra_init_worker_by_lua);
+
+    # note that it's different from APISIX.pm,
+    # here we enable no_error_log ignoreless of error_log.
+    if (!$block->no_error_log) {
+        $block->set_value("no_error_log", "[error]");
+    }
+
+    if (!$block->timeout) {
+        $block->set_value("timeout", "10");
+    }
+});
+
+add_cleanup_handler(sub {
+    unlink("/tmp/apisix_inspect_hooks.lua");
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: simple hook
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run1")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("t/lib/test_inspect.lua", 27, nil, function(info)
+                ngx.log(ngx.WARN, "var1=", info.vals.var1)
+                return true
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            do_request()
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+        }
+    }
+--- error_log
+var1=hello
+
+
+
+=== TEST 2: filename only
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run1")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("test_inspect.lua", 27, nil, function(info)
+                ngx.log(ngx.WARN, "var1=", info.vals.var1)
+                return true
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            do_request()
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+        }
+    }
+--- error_log
+var1=hello
+
+
+
+=== TEST 3: hook lifetime
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run1")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            local hook1_times = 2
+            dbg.set_hook("test_inspect.lua", 27, nil, function(info)
+                ngx.log(ngx.WARN, "var1=", info.vals.var1)
+                hook1_times = hook1_times - 1
+                return hook1_times == 0
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            -- request 3 times, but hook triggered 2 times
+            for _ = 1,3 do
+                do_request()
+            end
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+        }
+    }
+--- error_log
+var1=hello
+var1=hello
+
+
+
+=== TEST 4: multiple hooks
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run1")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("test_inspect.lua", 26, nil, function(info)
+                ngx.log(ngx.WARN, "var1=", info.vals.var1)
+                return true
+            end)
+
+            dbg.set_hook("test_inspect.lua", 27, nil, function(info)
+                ngx.log(ngx.WARN, "var2=", info.vals.var2)
+                return true
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            do_request()
+
+            -- note that we don't remove the hook file,
+            -- used for next test case
+        }
+    }
+--- error_log
+var1=hello
+var2=world
+
+
+
+=== TEST 5: hook file not removed, re-enabled by next startup
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run1")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            do_request()
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+        }
+    }
+--- error_log
+var1=hello
+
+
+
+=== TEST 6: soft link
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run1")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("t/lib/test_inspect.lua", 27, nil, function(info)
+                ngx.log(ngx.WARN, "var1=", info.vals.var1)
+                return true
+            end)
+            ]], "/tmp/test_real_tmp_file.lua")
+
+            os.execute("ln -sf /tmp/test_real_tmp_file.lua /tmp/apisix_inspect_hooks.lua")
+
+            ngx.sleep(1.5)
+
+            do_request()
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+            os.remove("/tmp/test_real_tmp_file.lua")
+        }
+    }
+--- error_log
+var1=hello
+
+
+
+=== TEST 7: remove soft link would disable hooks
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run1")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("t/lib/test_inspect.lua", 27, nil, function(info)
+                ngx.log(ngx.WARN, "var1=", info.vals.var1)
+                return true
+            end)
+            ]], "/tmp/test_real_tmp_file.lua")
+
+            os.execute("ln -sf /tmp/test_real_tmp_file.lua /tmp/apisix_inspect_hooks.lua")
+
+            ngx.sleep(1.5)
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+            ngx.sleep(1.5)
+
+            do_request()
+
+            os.remove("/tmp/test_real_tmp_file.lua")
+        }
+    }
+--- no_error_log
+var1=hello
+
+
+
+=== TEST 8: ensure we see all local variables till the hook line
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run1")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("t/lib/test_inspect.lua", 27, nil, function(info)
+                local count = 0
+                for k,v in pairs(info.vals) do
+                    count = count + 1
+                end
+                ngx.log(ngx.WARN, "count=", count)
+                return true
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            do_request()
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+        }
+    }
+--- error_log
+count=2
+
+
+
+=== TEST 9: check upvalue of run2(), only upvalue used in function code are visible
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run2")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("t/lib/test_inspect.lua", 33, nil, function(info)
+                ngx.log(ngx.WARN, "upvar1=", info.uv.upvar1)
+                ngx.log(ngx.WARN, "upvar2=", info.uv.upvar2)
+                return true
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            do_request()
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+        }
+    }
+--- error_log
+upvar1=2
+upvar2=nil
+
+
+
+=== TEST 10: check upvalue of run3(), now both upvar1 and upvar2 are visible
+--- config
+    location /t {
+        content_by_lua_block {
+            local code = set_test_route("run3")
+            if code >= 300 then
+                ngx.status = code
+                return
+            end
+
+            write_hooks([[
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("t/lib/test_inspect.lua", 37, nil, function(info)
+                ngx.log(ngx.WARN, "upvar1=", info.uv.upvar1)
+                ngx.log(ngx.WARN, "upvar2=", info.uv.upvar2)
+                return true
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            do_request()
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+        }
+    }
+--- error_log
+upvar1=2
+upvar2=yes
+
+
+
+=== TEST 11: flush specific JIT cache
+--- config
+    location /t {
+        content_by_lua_block {
+            local test = require("lib.test_inspect")
+
+            local t1 = test.hot1()
+            local t8 = test.hot2()
+
+            write_hooks([[
+            local test = require("lib.test_inspect")
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("t/lib/test_inspect.lua", 47, test.hot1, function(info)
+                return false
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            local t2 = test.hot1()
+            local t9 = test.hot2()
+
+            assert(t2-t1 > t1, "hot1 consumes at least double times than before")
+            assert(t9-t8 < t8*0.8, "hot2 not affected")
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+
+            ngx.sleep(1.5)
+
+            local t3 = test.hot1()
+            local t4 = test.hot2()
+            assert(t3-t1 < t1*0.8, "hot1 jit recover")
+            assert(t4-t8 < t4*0.8, "hot2 jit recover")
+        }
+    }
+
+
+
+=== TEST 12: flush the whole JIT cache
+--- config
+    location /t {
+        content_by_lua_block {
+            local test = require("lib.test_inspect")
+
+            local t1 = test.hot1()
+            local t8 = test.hot2()
+
+            write_hooks([[
+            local test = require("lib.test_inspect")
+            local dbg = require "apisix.inspect.dbg"
+            dbg.set_hook("t/lib/test_inspect.lua", 47, nil, function(info)
+                return false
+            end)
+            ]])
+
+            ngx.sleep(1.5)
+
+            local t2 = test.hot1()
+            local t9 = test.hot2()
+
+            assert(t2-t1 > t1, "hot1 consumes at least double times than before")
+            assert(t9-t8 > t8, "hot2 consumes at least double times than before")
+
+            os.remove("/tmp/apisix_inspect_hooks.lua")
+
+            ngx.sleep(1.5)
+
+            local t3 = test.hot1()
+            local t4 = test.hot2()
+            assert(t3-t1 < t1*0.8, "hot1 jit recover")
+            assert(t4-t8 < t4*0.8, "hot2 jit recover")
+        }
+    }