From ee0e7a2899a6ceb784eba475a8eb65e4733710a8 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 11 Sep 2022 01:56:34 +0200 Subject: [PATCH 1/5] chore(dependencies): add luasocket 3.1.0-1 --- .vscode/settings.json | 1 + rockspecs/awesomewm.d.tl-dev-1.rockspec | 1 + 2 files changed, 2 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 226142b..8d007a6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,6 +18,7 @@ "luacheckrc", "lualogging", "Luarocks", + "luasocket", "mkdir", "setopt", "Stylua", diff --git a/rockspecs/awesomewm.d.tl-dev-1.rockspec b/rockspecs/awesomewm.d.tl-dev-1.rockspec index a846424..6e415cb 100644 --- a/rockspecs/awesomewm.d.tl-dev-1.rockspec +++ b/rockspecs/awesomewm.d.tl-dev-1.rockspec @@ -16,6 +16,7 @@ dependencies = { "htmlparser 0.3.9", "web_sanitize 1.3.0", "penlight 1.13.1", + "luasocket 3.1.0-1", } build = { type = "builtin", From fa445221dc44e7ad06f5a9a1bbefe64255b8e31b Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 11 Sep 2022 02:23:10 +0200 Subject: [PATCH 2/5] chore(dependencies): luasec 1.2.0-1 LuaSocket HTTP module needs an additional dependency to LuaSec to support TLS/SSL communications. (Required to perform HTTPS requests) --- .vscode/settings.json | 1 + rockspecs/awesomewm.d.tl-dev-1.rockspec | 1 + 2 files changed, 2 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 8d007a6..15c126e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,6 +18,7 @@ "luacheckrc", "lualogging", "Luarocks", + "luasec", "luasocket", "mkdir", "setopt", diff --git a/rockspecs/awesomewm.d.tl-dev-1.rockspec b/rockspecs/awesomewm.d.tl-dev-1.rockspec index 6e415cb..21cfb48 100644 --- a/rockspecs/awesomewm.d.tl-dev-1.rockspec +++ b/rockspecs/awesomewm.d.tl-dev-1.rockspec @@ -17,6 +17,7 @@ dependencies = { "web_sanitize 1.3.0", "penlight 1.13.1", "luasocket 3.1.0-1", + "luasec 1.2.0-1", } build = { type = "builtin", From ce92ce7fbb3279b9c7e44b13369bcfd09c034a09 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 11 Sep 2022 02:24:03 +0200 Subject: [PATCH 3/5] feat(crawler): migrate to LuaSocket --- src/awesomewm.d.tl/crawler/init.lua | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/awesomewm.d.tl/crawler/init.lua b/src/awesomewm.d.tl/crawler/init.lua index 5482bf1..f2a02d4 100644 --- a/src/awesomewm.d.tl/crawler/init.lua +++ b/src/awesomewm.d.tl/crawler/init.lua @@ -1,33 +1,28 @@ -local curl = require "cURL" +local http = require "socket.http" local inspect = require "inspect" local log = require "logger" local crawler = {} function crawler.request(url) - local queue = {} - local easy = curl.easy():setopt_url(url):setopt_writefunction(function(buf) - table.insert(queue, buf) - end) + local body, code_or_error = http.request(url) - local ok, err = easy:perform() - if not ok then - easy:close() - error { message = "curl.easy failed", err = err } + if not body then + local err = code_or_error + error { "Request failed", err = err, url = url } end - local code, body = easy:getinfo_response_code(), table.concat(queue) - easy:close() + local code = code_or_error if code < 200 and code >= 300 then error { - message = "curl response code is not 200", + "Request successful, but returned non-200 code", code = code, body = body, } end - return queue + return body end function crawler.fetch(url) @@ -40,7 +35,7 @@ function crawler.fetch(url) log:info(inspect { message = "Successfully fetched resource", url = url }) - return table.concat(result, "") + return result end return crawler From 5806f894d0019227b186bd7123421e3aa3f68dc2 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 11 Sep 2022 02:24:33 +0200 Subject: [PATCH 4/5] chore(dependencies): remove lua-curl --- rockspecs/awesomewm.d.tl-dev-1.rockspec | 1 - 1 file changed, 1 deletion(-) diff --git a/rockspecs/awesomewm.d.tl-dev-1.rockspec b/rockspecs/awesomewm.d.tl-dev-1.rockspec index 21cfb48..d91ae98 100644 --- a/rockspecs/awesomewm.d.tl-dev-1.rockspec +++ b/rockspecs/awesomewm.d.tl-dev-1.rockspec @@ -12,7 +12,6 @@ dependencies = { "lualogging 1.6.0", "inspect 3.1.3", "ansicolors 1.0.2", - "lua-curl 0.3.13", "htmlparser 0.3.9", "web_sanitize 1.3.0", "penlight 1.13.1", From 10c9c893b1955e4eac3f6c46fc065ed81b03cda7 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 11 Sep 2022 03:17:40 +0200 Subject: [PATCH 5/5] feat(crawler): manage `file://` protocol --- .vscode/settings.json | 1 + src/awesomewm.d.tl/crawler/init.lua | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 15c126e..18f3e6c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,6 +23,7 @@ "mkdir", "setopt", "Stylua", + "tablex", "tmpl", "wibox", "writefunction" diff --git a/src/awesomewm.d.tl/crawler/init.lua b/src/awesomewm.d.tl/crawler/init.lua index f2a02d4..1305b9c 100644 --- a/src/awesomewm.d.tl/crawler/init.lua +++ b/src/awesomewm.d.tl/crawler/init.lua @@ -1,10 +1,12 @@ +local file = require "pl.file" local http = require "socket.http" local inspect = require "inspect" local log = require "logger" +local tablex = require "pl.tablex" local crawler = {} -function crawler.request(url) +local function http_request(url) local body, code_or_error = http.request(url) if not body then @@ -25,8 +27,26 @@ function crawler.request(url) return body end +local function get_resource_by_protocol(url) + local protocol, resource = url:match "^(%a+)://(.*)$" + + if not not tablex.find({ "http", "https" }, protocol) then + local success, result = pcall(http_request, url) + return success, result + elseif protocol == "file" then + local res, err = file.read(resource, false) + if res then + return true, res + else + return false, err + end + else + error { "Unsupported protocol", protocol = protocol } + end +end + function crawler.fetch(url) - local success, result = pcall(crawler.request, url) + local success, result = get_resource_by_protocol(url) if not success then log:error(inspect { "Fetch failed", status = success, error = result })