feat(crawler): migrate to LuaSocket

This commit is contained in:
Aire-One 2022-09-11 02:24:03 +02:00
parent fa445221dc
commit ce92ce7fbb
1 changed files with 9 additions and 14 deletions

View File

@ -1,33 +1,28 @@
local curl = require "cURL" local http = require "socket.http"
local inspect = require "inspect" local inspect = require "inspect"
local log = require "logger" local log = require "logger"
local crawler = {} local crawler = {}
function crawler.request(url) function crawler.request(url)
local queue = {} local body, code_or_error = http.request(url)
local easy = curl.easy():setopt_url(url):setopt_writefunction(function(buf)
table.insert(queue, buf)
end)
local ok, err = easy:perform() if not body then
if not ok then local err = code_or_error
easy:close() error { "Request failed", err = err, url = url }
error { message = "curl.easy failed", err = err }
end end
local code, body = easy:getinfo_response_code(), table.concat(queue) local code = code_or_error
easy:close()
if code < 200 and code >= 300 then if code < 200 and code >= 300 then
error { error {
message = "curl response code is not 200", "Request successful, but returned non-200 code",
code = code, code = code,
body = body, body = body,
} }
end end
return queue return body
end end
function crawler.fetch(url) function crawler.fetch(url)
@ -40,7 +35,7 @@ function crawler.fetch(url)
log:info(inspect { message = "Successfully fetched resource", url = url }) log:info(inspect { message = "Successfully fetched resource", url = url })
return table.concat(result, "") return result
end end
return crawler return crawler