feat(crawler): migrate to LuaSocket
This commit is contained in:
parent
fa445221dc
commit
ce92ce7fbb
|
@ -1,33 +1,28 @@
|
||||||
local curl = require "cURL"
|
local http = require "socket.http"
|
||||||
local inspect = require "inspect"
|
local inspect = require "inspect"
|
||||||
local log = require "logger"
|
local log = require "logger"
|
||||||
|
|
||||||
local crawler = {}
|
local crawler = {}
|
||||||
|
|
||||||
function crawler.request(url)
|
function crawler.request(url)
|
||||||
local queue = {}
|
local body, code_or_error = http.request(url)
|
||||||
local easy = curl.easy():setopt_url(url):setopt_writefunction(function(buf)
|
|
||||||
table.insert(queue, buf)
|
|
||||||
end)
|
|
||||||
|
|
||||||
local ok, err = easy:perform()
|
if not body then
|
||||||
if not ok then
|
local err = code_or_error
|
||||||
easy:close()
|
error { "Request failed", err = err, url = url }
|
||||||
error { message = "curl.easy failed", err = err }
|
|
||||||
end
|
end
|
||||||
|
|
||||||
local code, body = easy:getinfo_response_code(), table.concat(queue)
|
local code = code_or_error
|
||||||
easy:close()
|
|
||||||
|
|
||||||
if code < 200 and code >= 300 then
|
if code < 200 and code >= 300 then
|
||||||
error {
|
error {
|
||||||
message = "curl response code is not 200",
|
"Request successful, but returned non-200 code",
|
||||||
code = code,
|
code = code,
|
||||||
body = body,
|
body = body,
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
return queue
|
return body
|
||||||
end
|
end
|
||||||
|
|
||||||
function crawler.fetch(url)
|
function crawler.fetch(url)
|
||||||
|
@ -40,7 +35,7 @@ function crawler.fetch(url)
|
||||||
|
|
||||||
log:info(inspect { message = "Successfully fetched resource", url = url })
|
log:info(inspect { message = "Successfully fetched resource", url = url })
|
||||||
|
|
||||||
return table.concat(result, "")
|
return result
|
||||||
end
|
end
|
||||||
|
|
||||||
return crawler
|
return crawler
|
||||||
|
|
Loading…
Reference in New Issue