You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by sp...@apache.org on 2022/05/25 02:10:32 UTC
[apisix] branch master updated: fix: reduce memory usage when abnormal weights are given in chash (#7103)
This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new dd7bfa7e8 fix: reduce memory usage when abnormal weights are given in chash (#7103)
dd7bfa7e8 is described below
commit dd7bfa7e89742968a9afdcf642bd984e7feb1c12
Author: 罗泽轩 <sp...@gmail.com>
AuthorDate: Wed May 25 10:10:24 2022 +0800
fix: reduce memory usage when abnormal weights are given in chash (#7103)
* fix: reduce memory usage when abnormal weights are given in chash
Unlike the planned, I choose to do the gcd in APISIX because in this way
we don't need to take care of dynamically resize.
Signed-off-by: spacewander <sp...@gmail.com>
* Update apisix/core/math.lua
Co-authored-by: tzssangglass <tz...@gmail.com>
Co-authored-by: tzssangglass <tz...@gmail.com>
---
apisix/balancer/chash.lua | 16 ++++++
apisix/core.lua | 1 +
apisix/core/dns/client.lua | 10 +---
apisix/core/math.lua | 41 +++++++++++++++
t/node/chash-balance.t | 124 +++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 183 insertions(+), 9 deletions(-)
diff --git a/apisix/balancer/chash.lua b/apisix/balancer/chash.lua
index b191407d2..f0e971a36 100644
--- a/apisix/balancer/chash.lua
+++ b/apisix/balancer/chash.lua
@@ -71,11 +71,27 @@ function _M.new(up_nodes, upstream)
local nodes_count = 0
local safe_limit = 0
+ local gcd = 0
local servers, nodes = {}, {}
+
+ for serv, weight in pairs(up_nodes) do
+ if gcd == 0 then
+ gcd = weight
+ else
+ gcd = core.math.gcd(gcd, weight)
+ end
+ end
+
+ if gcd == 0 then
+ -- all nodes' weight are 0
+ gcd = 1
+ end
+
for serv, weight in pairs(up_nodes) do
local id = str_gsub(serv, ":", str_null)
nodes_count = nodes_count + 1
+ weight = weight / gcd
safe_limit = safe_limit + weight
servers[id] = serv
nodes[id] = weight
diff --git a/apisix/core.lua b/apisix/core.lua
index e03996900..0e0919bb8 100644
--- a/apisix/core.lua
+++ b/apisix/core.lua
@@ -53,4 +53,5 @@ return {
resolver = require("apisix.core.resolver"),
os = require("apisix.core.os"),
pubsub = require("apisix.core.pubsub"),
+ math = require("apisix.core.math"),
}
diff --git a/apisix/core/dns/client.lua b/apisix/core/dns/client.lua
index b9dcfb9c8..1bf2aca4d 100644
--- a/apisix/core/dns/client.lua
+++ b/apisix/core/dns/client.lua
@@ -24,6 +24,7 @@ local config_local = require("apisix.core.config_local")
local log = require("apisix.core.log")
local json = require("apisix.core.json")
local table = require("apisix.core.table")
+local gcd = require("apisix.core.math").gcd
local insert_tab = table.insert
local math_random = math.random
local package_loaded = package.loaded
@@ -38,15 +39,6 @@ local _M = {
}
-local function gcd(a, b)
- if b == 0 then
- return a
- end
-
- return gcd(b, a % b)
-end
-
-
local function resolve_srv(client, answers)
if #answers == 0 then
return nil, "empty SRV record"
diff --git a/apisix/core/math.lua b/apisix/core/math.lua
new file mode 100644
index 000000000..1514cf7f0
--- /dev/null
+++ b/apisix/core/math.lua
@@ -0,0 +1,41 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--- Common library about math
+--
+-- @module core.math
+local _M = {}
+
+
+---
+-- Calculate the greatest common divisor (GCD) of two numbers
+--
+-- @function core.math.gcd
+-- @tparam number a
+-- @tparam number b
+-- @treturn number the GCD of a and b
+local function gcd(a, b)
+ if b == 0 then
+ return a
+ end
+
+ return gcd(b, a % b)
+end
+_M.gcd = gcd
+
+
+return _M
diff --git a/t/node/chash-balance.t b/t/node/chash-balance.t
index dfde2aaff..0c846e27e 100644
--- a/t/node/chash-balance.t
+++ b/t/node/chash-balance.t
@@ -556,3 +556,127 @@ passed
GET /t
--- response_body
200
+
+
+
+=== TEST 15: set routes with very big weights
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/server_port",
+ "upstream": {
+ "key": "arg_device_id",
+ "type": "chash",
+ "nodes": {
+ "127.0.0.1:1980": 1000000000,
+ "127.0.0.1:1981": 2000000000,
+ "127.0.0.1:1982": 1000000000
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 16: hit
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/server_port?device_id=1"
+
+ local httpc = http.new()
+ local res, err = httpc:request_uri(uri, {method = "GET"})
+ if not res then
+ ngx.say(err)
+ return
+ end
+
+ -- a `size too large` error will be thrown if we don't reduce the weight
+ ngx.say(res.status)
+ }
+ }
+--- request
+GET /t
+--- response_body
+200
+
+
+
+=== TEST 17: set routes with very big weights, some nodes have zero weight
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "uri": "/server_port",
+ "upstream": {
+ "key": "arg_device_id",
+ "type": "chash",
+ "nodes": {
+ "127.0.0.1:1980": 1000000000,
+ "127.0.0.1:1981": 0,
+ "127.0.0.1:1982": 4000000000
+ }
+ }
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- request
+GET /t
+--- response_body
+passed
+--- no_error_log
+[error]
+
+
+
+=== TEST 18: hit
+--- config
+ location /t {
+ content_by_lua_block {
+ local http = require "resty.http"
+ local uri = "http://127.0.0.1:" .. ngx.var.server_port
+ .. "/server_port?device_id=1"
+
+ local httpc = http.new()
+ local res, err = httpc:request_uri(uri, {method = "GET"})
+ if not res then
+ ngx.say(err)
+ return
+ end
+
+ -- a `size too large` error will be thrown if we don't reduce the weight
+ ngx.say(res.status)
+ }
+ }
+--- request
+GET /t
+--- response_body
+200