feat(crawler): migrate to LuaSocket
This commit is contained in:
parent
fa445221dc
commit
ce92ce7fbb
|
@ -1,33 +1,28 @@
|
|||
local curl = require "cURL"
|
||||
local http = require "socket.http"
|
||||
local inspect = require "inspect"
|
||||
local log = require "logger"
|
||||
|
||||
local crawler = {}
|
||||
|
||||
function crawler.request(url)
|
||||
local queue = {}
|
||||
local easy = curl.easy():setopt_url(url):setopt_writefunction(function(buf)
|
||||
table.insert(queue, buf)
|
||||
end)
|
||||
local body, code_or_error = http.request(url)
|
||||
|
||||
local ok, err = easy:perform()
|
||||
if not ok then
|
||||
easy:close()
|
||||
error { message = "curl.easy failed", err = err }
|
||||
if not body then
|
||||
local err = code_or_error
|
||||
error { "Request failed", err = err, url = url }
|
||||
end
|
||||
|
||||
local code, body = easy:getinfo_response_code(), table.concat(queue)
|
||||
easy:close()
|
||||
local code = code_or_error
|
||||
|
||||
if code < 200 and code >= 300 then
|
||||
error {
|
||||
message = "curl response code is not 200",
|
||||
"Request successful, but returned non-200 code",
|
||||
code = code,
|
||||
body = body,
|
||||
}
|
||||
end
|
||||
|
||||
return queue
|
||||
return body
|
||||
end
|
||||
|
||||
function crawler.fetch(url)
|
||||
|
@ -40,7 +35,7 @@ function crawler.fetch(url)
|
|||
|
||||
log:info(inspect { message = "Successfully fetched resource", url = url })
|
||||
|
||||
return table.concat(result, "")
|
||||
return result
|
||||
end
|
||||
|
||||
return crawler
|
||||
|
|
Loading…
Reference in New Issue