You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@apisix.apache.org by GitBox <gi...@apache.org> on 2022/05/29 13:25:03 UTC
[GitHub] [apisix] spacewander commented on a diff in pull request #7099: init elasticsearch-logger.lua
spacewander commented on code in PR #7099:
URL: https://github.com/apache/apisix/pull/7099#discussion_r884270602
##########
t/plugin/elasticsearch-logger.t:
##########
@@ -0,0 +1,182 @@
+#
Review Comment:
Let's add `elasticsearch-logger` to https://github.com/apache/apisix/blob/9b0cc7a3b0ed837e330603304c4a7267ff392faf/conf/config-default.yaml#L324 so CI can be run
##########
t/plugin/elasticsearch-logger.t:
##########
@@ -0,0 +1,182 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+ $block->set_value("no_error_log", "[error]");
+ }
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $http_config = $block->http_config // <<_EOC_;
+ server {
+ listen 10420;
+ location /elasticsearch-logger/test {
Review Comment:
The URL should contain the index name?
##########
t/plugin/elasticsearch-logger.t:
##########
@@ -0,0 +1,182 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+log_level("info");
+repeat_each(1);
+no_long_string();
+no_root_location();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+ $block->set_value("no_error_log", "[error]");
+ }
+
+ if (!defined $block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ my $http_config = $block->http_config // <<_EOC_;
+ server {
+ listen 10420;
+ location /elasticsearch-logger/test {
+ content_by_lua_block {
+ ngx.req.read_body()
+ local data = ngx.req.get_body_data()
+ local headers = ngx.req.get_headers()
+ ngx.log(ngx.WARN, "elasticsearch body: ", data)
+ for k, v in pairs(headers) do
+ ngx.log(ngx.WARN, "elasticsearch headers: " .. k .. ":" .. v)
+ end
+ ngx.say("ok")
+ }
+ }
+ }
+_EOC_
+
+ $block->set_value("http_config", $http_config);
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: Full configuration verification
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.elasticsearch-logger")
+ local ok, err = plugin.check_schema({timeout = 3,
+ retry_delay = 1,
+ batch_max_size = 500,
+ user = "elastic",
+ password = "elastic",
+ index = "apisix-elasticsearch-log",
+ endpoint_addr = "http://127.0.0.1:10420/elasticsearch-logger/test",
+ max_retry_count = 1,
+ name = "elasticsearch logger",
+ ssl_verify = false
+ })
+
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("passed")
+ end
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 2: Basic configuration verification
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.elasticsearch-logger")
+ local ok, err = plugin.check_schema({user = "elastic",
+ password = "elastic",
+ index = "apisix-elasticsearch-log",
+ endpoint_addr = "http://127.0.0.1:10420/elasticsearch-logger/test"
+ })
+
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("passed")
+ end
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 3: auth configure undefined
+--- config
+ location /t {
+ content_by_lua_block {
+ local plugin = require("apisix.plugins.elasticsearch-logger")
+ local ok, err = plugin.check_schema({user = "elastic",
+ password = "elastic",
+ index = "apisix-elasticsearch-log",
+ })
+
+ if not ok then
+ ngx.say(err)
+ else
+ ngx.say("passed")
+ end
+ }
+ }
+--- response_body
+property "endpoint_addr" is required
+
+
+
+=== TEST 4: add plugin on routes
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+ local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+ "plugins": {
+ "elasticsearch-logger": {
+ "user": "elastic",
+ "password": "elastic",
+ "index": "apisix-elasticsearch-log",
+ "endpoint_addr": "http://127.0.0.1:10420/elasticsearch-logger/test",
+ "batch_max_size":1,
+ "inactive_timeout":1
+ }
+ },
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:5601": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/opentracing"
+ }]]
+ )
+
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 5: access local server
+--- request
+GET /opentracing
+--- response_body
+opentracing
+--- error_log
+elasticsearch body:
Review Comment:
We need to check the upload data in the body
##########
apisix/plugins/elasticsearch-logger.lua:
##########
@@ -0,0 +1,181 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local bp_manager_mod = require("apisix.utils.batch-processor-manager")
+local log_util = require("apisix.utils.log-util")
+local core = require("apisix.core")
+local http = require("resty.http")
+local url = require("net.url")
+local plugin = require("apisix.plugin")
+
+local ngx = ngx
+local tostring = tostring
+local encode_base64 = ngx.encode_base64
+
+local plugin_name = "elasticsearch-logger"
+local batch_processor_manager = bp_manager_mod.new(plugin_name)
+
+local schema = {
+ type = "object",
+ properties = {
+ endpoint_addr = core.schema.uri_def,
+ user = {type = "string", default = ""},
+ password = {type = "string", default = ""},
+ index = {type = "string", default = ""},
+ timeout = {type = "integer", minimum = 1, default = 3},
+ name = {type = "string", default = "elasticsearch logger"},
+ ssl_verify = {type = "boolean", default = true},
+ },
+ required = {"endpoint_addr", "user", "password", "index"}
+}
+
+
+local metadata_schema = {
+ type = "object",
+ properties = {
+ log_format = log_util.metadata_schema_log_format,
+ },
+}
+
+
+local _M = {
+ version = 0.1,
+ priority = 398,
+ name = plugin_name,
+ schema = batch_processor_manager:wrap_schema(schema),
+ metadata_schema = metadata_schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+ if schema_type == core.schema.TYPE_METADATA then
+ return core.schema.check(metadata_schema, conf)
+ end
+ return core.schema.check(schema, conf)
+end
+
+
+local function send_http_data(conf, log_message)
+ local err_msg
+ local res = true
+ local url_decoded = url.parse(conf.endpoint_addr)
+ local host = url_decoded.host
+ local port = url_decoded.port
+ local index = conf.index .. "-" .. os.date('%Y.%m.%d')
Review Comment:
The `conf.index` can't be empty?
##########
apisix/plugins/elasticsearch-logger.lua:
##########
@@ -0,0 +1,181 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local bp_manager_mod = require("apisix.utils.batch-processor-manager")
+local log_util = require("apisix.utils.log-util")
+local core = require("apisix.core")
+local http = require("resty.http")
+local url = require("net.url")
+local plugin = require("apisix.plugin")
+
+local ngx = ngx
+local tostring = tostring
+local encode_base64 = ngx.encode_base64
+
+local plugin_name = "elasticsearch-logger"
+local batch_processor_manager = bp_manager_mod.new(plugin_name)
+
+local schema = {
+ type = "object",
+ properties = {
+ endpoint_addr = core.schema.uri_def,
+ user = {type = "string", default = ""},
+ password = {type = "string", default = ""},
+ index = {type = "string", default = ""},
+ timeout = {type = "integer", minimum = 1, default = 3},
+ name = {type = "string", default = "elasticsearch logger"},
+ ssl_verify = {type = "boolean", default = true},
+ },
+ required = {"endpoint_addr", "user", "password", "index"}
+}
+
+
+local metadata_schema = {
+ type = "object",
+ properties = {
+ log_format = log_util.metadata_schema_log_format,
+ },
+}
+
+
+local _M = {
+ version = 0.1,
+ priority = 398,
+ name = plugin_name,
+ schema = batch_processor_manager:wrap_schema(schema),
+ metadata_schema = metadata_schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+ if schema_type == core.schema.TYPE_METADATA then
+ return core.schema.check(metadata_schema, conf)
+ end
+ return core.schema.check(schema, conf)
+end
+
+
+local function send_http_data(conf, log_message)
+ local err_msg
+ local res = true
+ local url_decoded = url.parse(conf.endpoint_addr)
+ local host = url_decoded.host
+ local port = url_decoded.port
+ local index = conf.index .. "-" .. os.date('%Y.%m.%d')
+
+ core.log.info("sending a batch logs to ", conf.endpoint_addr .. index)
+
+ if not port then
+ if url_decoded.scheme == "https" then
+ port = 443
+ else
+ port = 80
+ end
+ end
+
+ local httpc = http.new()
+ httpc:set_timeout(conf.timeout * 1000)
+ local ok, err = httpc:connect(host, port)
+
+ if not ok then
+ return false, "failed to connect to host[" .. host .. "] port["
+ .. tostring(port) .. "] " .. err
+ end
+
+ if url_decoded.scheme == "https" then
+ ok, err = httpc:ssl_handshake(true, host, conf.ssl_verify)
+ if not ok then
+ return false, "failed to perform SSL with host[" .. host .. "] "
+ .. "port[" .. tostring(port) .. "] " .. err
+ end
+ end
+
+ local url_path = index .. "/_doc"
+ local user_pass = conf.user .. ":" .. conf.password
Review Comment:
Should bypass the authentication if conf.user is empty?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@apisix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org