You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by GitBox <gi...@apache.org> on 2021/07/09 02:09:54 UTC
[GitHub] [apisix] spacewander commented on a change in pull request #4559: feat: Request-ID plugin add snowflake algorithm
spacewander commented on a change in pull request #4559:
URL: https://github.com/apache/apisix/pull/4559#discussion_r666608780
##########
File path: apisix/plugins/request-id.lua
##########
@@ -14,36 +14,163 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-local core = require("apisix.core")
-local plugin_name = "request-id"
-local ngx = ngx
-local uuid = require("resty.jit-uuid")
+
+local ngx = ngx
+local bit = require("bit")
+local core = require("apisix.core")
+local snowflake = require("snowflake")
+local uuid = require("resty.jit-uuid")
+local process = require("ngx.process")
+local tostring = tostring
+local math_pow = math.pow
+
+local plugin_name = "request-id"
+
+local worker_number = nil
+local snowflake_init = nil
+
+local attr = nil
local schema = {
type = "object",
properties = {
header_name = {type = "string", default = "X-Request-Id"},
- include_in_response = {type = "boolean", default = true}
+ include_in_response = {type = "boolean", default = true},
+ algorithm = {type = "string", enum = {"uuid", "snowflake"}, default = "uuid"}
}
}
+local attr_schema = {
+ type = "object",
+ properties = {
+ snowflake = {
+ type = "object",
+ properties = {
+ enable = {type = "boolean"},
+ snowflake_epoc = {type = "integer", minimum = 1, default = 1609459200000},
+ node_id_bits = {type = "integer", minimum = 1, default = 5},
+ sequence_bits = {type = "integer", minimum = 1, default = 10},
+ datacenter_id_bits = {type = "integer", minimum = 1, default = 5},
Review comment:
Better to use a user-friendly name, like max_worker_number or ids_per_second
##########
File path: apisix/plugins/request-id.lua
##########
@@ -65,4 +191,47 @@ function _M.header_filter(conf, ctx)
end
end
+function _M.init()
+ local local_conf = core.config.local_conf()
+ attr = core.table.try_read_attr(local_conf, "plugin_attr", plugin_name)
+ local ok, err = core.schema.check(attr_schema, attr)
+ if not ok then
+ core.log.error("failed to check the plugin_attr[", plugin_name, "]", ": ", err)
+ return
+ end
+ if attr.snowflake.enable then
+ if process.type() == "worker" then
+ ngx.timer.at(0, next_id)
+ end
+ end
+end
+
+function _M.api()
Review comment:
Why do we need to expose the uuid? This will leak the info about the mechanism that we use to generate the uuid.
##########
File path: apisix/plugins/request-id.lua
##########
@@ -14,36 +14,163 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-local core = require("apisix.core")
-local plugin_name = "request-id"
-local ngx = ngx
-local uuid = require("resty.jit-uuid")
+
+local ngx = ngx
+local bit = require("bit")
+local core = require("apisix.core")
+local snowflake = require("snowflake")
+local uuid = require("resty.jit-uuid")
+local process = require("ngx.process")
+local tostring = tostring
+local math_pow = math.pow
+
+local plugin_name = "request-id"
+
+local worker_number = nil
+local snowflake_init = nil
+
+local attr = nil
local schema = {
type = "object",
properties = {
header_name = {type = "string", default = "X-Request-Id"},
- include_in_response = {type = "boolean", default = true}
+ include_in_response = {type = "boolean", default = true},
+ algorithm = {type = "string", enum = {"uuid", "snowflake"}, default = "uuid"}
}
}
+local attr_schema = {
+ type = "object",
+ properties = {
+ snowflake = {
+ type = "object",
+ properties = {
+ enable = {type = "boolean"},
+ snowflake_epoc = {type = "integer", minimum = 1, default = 1609459200000},
+ node_id_bits = {type = "integer", minimum = 1, default = 5},
+ sequence_bits = {type = "integer", minimum = 1, default = 10},
+ datacenter_id_bits = {type = "integer", minimum = 1, default = 5},
+ worker_number_ttl = {type = "integer", minimum = 1, default = 30},
+ worker_number_interval = {type = "integer", minimum = 1, default = 10}
+ }
+ }
+ }
+}
local _M = {
version = 0.1,
priority = 11010,
name = plugin_name,
- schema = schema,
+ schema = schema
}
-
function _M.check_schema(conf)
return core.schema.check(schema, conf)
end
+local function gen_worker_number(max_number)
+ if worker_number == nil then
+ local etcd_cli, prefix = core.etcd.new()
+ local res, _ = etcd_cli:grant(attr.snowflake.worker_number_ttl)
+
+ local prefix = prefix .. "/plugins/request-id/snowflake/"
+ local uuid = uuid.generate_v4()
+ local id = 1
+ while (id <= max_number) do
+ ::continue::
+ local _, err1 = etcd_cli:setnx(prefix .. tostring(id), uuid)
+ local res2, err2 = etcd_cli:get(prefix .. tostring(id))
+
+ if err1 or err2 or res2.body.kvs[1].value ~= uuid then
+ core.log.notice("worker_number " .. id .. " is not available")
+ id = id + 1
+ else
+ worker_number = id
+
+ local _, err3 =
+ etcd_cli:set(
+ prefix .. tostring(id),
+ uuid,
+ {
+ prev_kv = true,
+ lease = res.body.ID
+ }
+ )
+
+ if err3 then
+ id = id + 1
+ etcd_cli:delete(prefix .. tostring(id))
+ core.log.error("set worker_number " .. id .. " lease error: " .. err3)
+ goto continue
+ end
+
+ local handler = function(premature, etcd_cli, lease_id)
+ local _, err4 = etcd_cli:keepalive(lease_id)
+ if err4 then
+ snowflake_init = nil
+ worker_number = nil
+ core.log.error("snowflake worker_number lease faild.")
+ end
+ core.log.info("snowflake worker_number lease success.")
+ end
+ ngx.timer.every(attr.snowflake.worker_number_interval,
+ handler, etcd_cli, res.body.ID)
+
+ core.log.notice("snowflake worker_number: " .. id)
+ break
+ end
+ end
+
+ if worker_number == nil then
+ core.log.error("No worker_number is not available")
+ return nil
+ end
+ end
+ return worker_number
+end
+
+local function split_worker_number(worker_number, node_id_bits, datacenter_id_bits)
+ local num = bit.tobit(worker_number)
+ local worker_id = bit.band(num, math_pow(2, node_id_bits) - 1) + 1
+ num = bit.rshift(num, node_id_bits)
+ local datacenter_id = bit.band(num, math_pow(2, datacenter_id_bits) - 1) + 1
+ return worker_id, datacenter_id
+end
+
+local function next_id()
+ if snowflake_init == nil then
+ local max_number = math_pow(2, (attr.snowflake.node_id_bits +
+ attr.snowflake.datacenter_id_bits))
+ worker_number = gen_worker_number(max_number)
+ if worker_number == nil then
+ return ""
+ end
+ local worker_id, datacenter_id = split_worker_number(worker_number,
+ attr.snowflake.node_id_bits, attr.snowflake.datacenter_id_bits)
+ core.log.notice("snowflake init datacenter_id: " ..
+ datacenter_id .. " worker_id: " .. worker_id)
+ snowflake.init(
+ worker_id,
+ datacenter_id,
+ attr.snowflake.snowflake_epoc,
+ attr.snowflake.node_id_bits,
+ attr.snowflake.datacenter_id_bits,
+ attr.snowflake.sequence_bits
+ )
+ snowflake_init = true
+ end
+ return snowflake:next_id()
+end
function _M.rewrite(conf, ctx)
local headers = ngx.req.get_headers()
- local uuid_val = uuid()
+ local uuid_val
Review comment:
Better to wrap it into a function
##########
File path: apisix/plugins/request-id.lua
##########
@@ -65,4 +191,47 @@ function _M.header_filter(conf, ctx)
end
end
+function _M.init()
+ local local_conf = core.config.local_conf()
+ attr = core.table.try_read_attr(local_conf, "plugin_attr", plugin_name)
+ local ok, err = core.schema.check(attr_schema, attr)
+ if not ok then
+ core.log.error("failed to check the plugin_attr[", plugin_name, "]", ": ", err)
+ return
+ end
+ if attr.snowflake.enable then
+ if process.type() == "worker" then
+ ngx.timer.at(0, next_id)
Review comment:
What about splitting the next_id into two functions, one for the init?
##########
File path: apisix/plugins/request-id.lua
##########
@@ -14,36 +14,163 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-local core = require("apisix.core")
-local plugin_name = "request-id"
-local ngx = ngx
-local uuid = require("resty.jit-uuid")
+
+local ngx = ngx
+local bit = require("bit")
+local core = require("apisix.core")
+local snowflake = require("snowflake")
+local uuid = require("resty.jit-uuid")
+local process = require("ngx.process")
+local tostring = tostring
+local math_pow = math.pow
+
+local plugin_name = "request-id"
+
+local worker_number = nil
+local snowflake_init = nil
+
+local attr = nil
local schema = {
type = "object",
properties = {
header_name = {type = "string", default = "X-Request-Id"},
- include_in_response = {type = "boolean", default = true}
+ include_in_response = {type = "boolean", default = true},
+ algorithm = {type = "string", enum = {"uuid", "snowflake"}, default = "uuid"}
}
}
+local attr_schema = {
+ type = "object",
+ properties = {
+ snowflake = {
+ type = "object",
+ properties = {
+ enable = {type = "boolean"},
+ snowflake_epoc = {type = "integer", minimum = 1, default = 1609459200000},
+ node_id_bits = {type = "integer", minimum = 1, default = 5},
+ sequence_bits = {type = "integer", minimum = 1, default = 10},
+ datacenter_id_bits = {type = "integer", minimum = 1, default = 5},
+ worker_number_ttl = {type = "integer", minimum = 1, default = 30},
+ worker_number_interval = {type = "integer", minimum = 1, default = 10}
+ }
+ }
+ }
+}
local _M = {
version = 0.1,
priority = 11010,
name = plugin_name,
- schema = schema,
+ schema = schema
}
-
function _M.check_schema(conf)
return core.schema.check(schema, conf)
end
+local function gen_worker_number(max_number)
+ if worker_number == nil then
+ local etcd_cli, prefix = core.etcd.new()
+ local res, _ = etcd_cli:grant(attr.snowflake.worker_number_ttl)
+
+ local prefix = prefix .. "/plugins/request-id/snowflake/"
+ local uuid = uuid.generate_v4()
+ local id = 1
+ while (id <= max_number) do
+ ::continue::
+ local _, err1 = etcd_cli:setnx(prefix .. tostring(id), uuid)
+ local res2, err2 = etcd_cli:get(prefix .. tostring(id))
+
+ if err1 or err2 or res2.body.kvs[1].value ~= uuid then
+ core.log.notice("worker_number " .. id .. " is not available")
+ id = id + 1
+ else
+ worker_number = id
+
+ local _, err3 =
+ etcd_cli:set(
+ prefix .. tostring(id),
+ uuid,
+ {
+ prev_kv = true,
+ lease = res.body.ID
+ }
+ )
+
+ if err3 then
+ id = id + 1
+ etcd_cli:delete(prefix .. tostring(id))
+ core.log.error("set worker_number " .. id .. " lease error: " .. err3)
+ goto continue
+ end
+
+ local handler = function(premature, etcd_cli, lease_id)
+ local _, err4 = etcd_cli:keepalive(lease_id)
+ if err4 then
+ snowflake_init = nil
+ worker_number = nil
+ core.log.error("snowflake worker_number lease faild.")
+ end
+ core.log.info("snowflake worker_number lease success.")
+ end
+ ngx.timer.every(attr.snowflake.worker_number_interval,
Review comment:
Can we write it via: https://github.com/apache/apisix/blob/89765ba188834c1b4c9f55088a28db25a959f39e/apisix/plugins/error-log-logger.lua#L198
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org