You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@community.apache.org by se...@apache.org on 2023/03/20 11:48:17 UTC
svn commit: r1908564 - in /comdev/helpwanted.apache.org/site: admin/elastic.lua lib/elastic.lua
Author: sebb
Date: Mon Mar 20 11:48:17 2023
New Revision: 1908564
URL: http://svn.apache.org/viewvc?rev=1908564&view=rev
Log:
Add queryscroll generator
Modified:
comdev/helpwanted.apache.org/site/admin/elastic.lua
comdev/helpwanted.apache.org/site/lib/elastic.lua
Modified: comdev/helpwanted.apache.org/site/admin/elastic.lua
URL: http://svn.apache.org/viewvc/comdev/helpwanted.apache.org/site/admin/elastic.lua?rev=1908564&r1=1908563&r2=1908564&view=diff
==============================================================================
--- comdev/helpwanted.apache.org/site/admin/elastic.lua (original)
+++ comdev/helpwanted.apache.org/site/admin/elastic.lua Mon Mar 20 11:48:17 2023
@@ -17,10 +17,20 @@
-- This is elastic.lua - ElasticSearch library
+-- N.B. Please keep lib/elastic.lua and admin/elastic.lua synchronised
+
+-- Note re http.request:
+-- In case of failure, the function returns nil followed by an error message.
+-- If successful, the simple form returns the response body as a string
+-- followed by the response status code, the response headers and the response status line.
+-- The generic function returns the same information,
+-- except the first return value is just the number 1 (the body goes to the sink).
+
local http = require 'socket.http'
local ltn12 = require 'ltn12'
local JSON = require 'cjson'
local config = {
+ es_host = "http://localhost:9200/",
es_url = "http://localhost:9200/helpwanted/"
}
local default_doc = "item"
@@ -28,20 +38,25 @@ local default_doc = "item"
-- ES no longer supports Content-type: application/x-www-form-urlencoded
-- which is the default with the simple interface http.request(url, body)
-- we have to use the rather more complicated generic interface
-local function _http_request(url, body)
- local hdrs = {}
- source=ltn12.source.string(body)
- hdrs['content-length'] = #body
- hdrs['content-type'] = 'application/json'
- local result = {}
- http.request{
- url = url,
- method = 'POST',
- headers = hdrs,
- sink=ltn12.sink.table(result),
- source=source
- }
- return table.concat(result)
+-- returns the body as the first parameter (if successful), emulating the simple interface
+local function _http_request(url, body, method)
+ local inhdrs = {}
+ source=ltn12.source.string(body)
+ inhdrs['content-length'] = #body
+ inhdrs['content-type'] = 'application/json'
+ local result = {}
+ local one, code, outhdrs, status = http.request {
+ url = url,
+ method = method or 'POST',
+ headers = inhdrs,
+ sink=ltn12.sink.table(result),
+ source=source
+ }
+ if one then -- success
+ return table.concat(result), code, outhdrs, status
+ else
+ return one, code
+ end
end
-- Standard ES query, returns $size results of any doc of type $doc, sorting by $sitem (desc)
@@ -86,6 +101,31 @@ local function raw(query, doctype)
return json or {}
end
+-- scrolling query generator
+local function scrollquerygen(query, doctype)
+ -- start off the scroll
+ local url = config.es_url .. (doctype or default_doc) .. "/_search?scroll=1m"
+ local result = _http_request(url, JSON.encode(query))
+ local js = JSON.decode(result)
+ while js and js.hits and js.hits.hits and #js.hits.hits > 0 do -- scroll as long as we get new results
+ coroutine.yield(js)
+ -- continue the scroll (N.B. does not use the doctype)
+ local url = config.es_host .. "_search/scroll?scroll=1m&scroll_id=" .. js._scroll_id
+ local result = http.request(url)
+ js = JSON.decode(result)
+ end
+ -- drop the scroll
+ local url = config.es_host .. "_search/scroll/" .. js._scroll_id
+ http.request {
+ url = url,
+ method = 'DELETE'
+ }
+end
+
+local function scrollquery(query, doctype)
+ return coroutine.wrap(function () scrollquerygen(query, doctype) end)
+end
+
-- Update a document
local function update(doctype, id, query)
local js = JSON.encode({doc = query })
@@ -112,6 +152,7 @@ return {
find = find,
get = get,
raw = raw,
+ scrollquery = scrollquery,
index = index,
update = update
}
Modified: comdev/helpwanted.apache.org/site/lib/elastic.lua
URL: http://svn.apache.org/viewvc/comdev/helpwanted.apache.org/site/lib/elastic.lua?rev=1908564&r1=1908563&r2=1908564&view=diff
==============================================================================
--- comdev/helpwanted.apache.org/site/lib/elastic.lua (original)
+++ comdev/helpwanted.apache.org/site/lib/elastic.lua Mon Mar 20 11:48:17 2023
@@ -17,10 +17,20 @@
-- This is elastic.lua - ElasticSearch library
+-- N.B. Please keep lib/elastic.lua and admin/elastic.lua synchronised
+
+-- Note re http.request:
+-- In case of failure, the function returns nil followed by an error message.
+-- If successful, the simple form returns the response body as a string
+-- followed by the response status code, the response headers and the response status line.
+-- The generic function returns the same information,
+-- except the first return value is just the number 1 (the body goes to the sink).
+
local http = require 'socket.http'
local ltn12 = require 'ltn12'
local JSON = require 'cjson'
local config = {
+ es_host = "http://localhost:9200/",
es_url = "http://localhost:9200/helpwanted/"
}
local default_doc = "item"
@@ -28,20 +38,25 @@ local default_doc = "item"
-- ES no longer supports Content-type: application/x-www-form-urlencoded
-- which is the default with the simple interface http.request(url, body)
-- we have to use the rather more complicated generic interface
-local function _http_request(url, body)
- local hdrs = {}
- source=ltn12.source.string(body)
- hdrs['content-length'] = #body
- hdrs['content-type'] = 'application/json'
- local result = {}
- http.request{
- url = url,
- method = 'POST',
- headers = hdrs,
- sink=ltn12.sink.table(result),
- source=source
- }
- return table.concat(result)
+-- returns the body as the first parameter (if successful), emulating the simple interface
+local function _http_request(url, body, method)
+ local inhdrs = {}
+ source=ltn12.source.string(body)
+ inhdrs['content-length'] = #body
+ inhdrs['content-type'] = 'application/json'
+ local result = {}
+ local one, code, outhdrs, status = http.request {
+ url = url,
+ method = method or 'POST',
+ headers = inhdrs,
+ sink=ltn12.sink.table(result),
+ source=source
+ }
+ if one then -- success
+ return table.concat(result), code, outhdrs, status
+ else
+ return one, code
+ end
end
-- Standard ES query, returns $size results of any doc of type $doc, sorting by $sitem (desc)
@@ -86,6 +101,31 @@ local function raw(query, doctype)
return json or {}
end
+-- scrolling query generator
+local function scrollquerygen(query, doctype)
+ -- start off the scroll
+ local url = config.es_url .. (doctype or default_doc) .. "/_search?scroll=1m"
+ local result = _http_request(url, JSON.encode(query))
+ local js = JSON.decode(result)
+ while js and js.hits and js.hits.hits and #js.hits.hits > 0 do -- scroll as long as we get new results
+ coroutine.yield(js)
+ -- continue the scroll (N.B. does not use the doctype)
+ local url = config.es_host .. "_search/scroll?scroll=1m&scroll_id=" .. js._scroll_id
+ local result = http.request(url)
+ js = JSON.decode(result)
+ end
+ -- drop the scroll
+ local url = config.es_host .. "_search/scroll/" .. js._scroll_id
+ http.request {
+ url = url,
+ method = 'DELETE'
+ }
+end
+
+local function scrollquery(query, doctype)
+ return coroutine.wrap(function () scrollquerygen(query, doctype) end)
+end
+
-- Update a document
local function update(doctype, id, query)
local js = JSON.encode({doc = query })
@@ -112,6 +152,7 @@ return {
find = find,
get = get,
raw = raw,
+ scrollquery = scrollquery,
index = index,
update = update
}