feat(crawler): migrate to LuaSocket

This commit is contained in:
Aire-One 2022-09-11 02:24:03 +02:00
parent fa445221dc
commit ce92ce7fbb
1 changed files with 9 additions and 14 deletions

View File

@ -1,33 +1,28 @@
local curl = require "cURL"
local http = require "socket.http"
local inspect = require "inspect"
local log = require "logger"
local crawler = {}
function crawler.request(url)
local queue = {}
local easy = curl.easy():setopt_url(url):setopt_writefunction(function(buf)
table.insert(queue, buf)
end)
local body, code_or_error = http.request(url)
local ok, err = easy:perform()
if not ok then
easy:close()
error { message = "curl.easy failed", err = err }
if not body then
local err = code_or_error
error { "Request failed", err = err, url = url }
end
local code, body = easy:getinfo_response_code(), table.concat(queue)
easy:close()
local code = code_or_error
if code < 200 and code >= 300 then
error {
message = "curl response code is not 200",
"Request successful, but returned non-200 code",
code = code,
body = body,
}
end
return queue
return body
end
function crawler.fetch(url)
@ -40,7 +35,7 @@ function crawler.fetch(url)
log:info(inspect { message = "Successfully fetched resource", url = url })
return table.concat(result, "")
return result
end
return crawler