diff --git a/src/awesomewm.d.tl/init.lua b/src/awesomewm.d.tl/init.lua
index decd5bb..686c1ee 100644
--- a/src/awesomewm.d.tl/init.lua
+++ b/src/awesomewm.d.tl/init.lua
@@ -3,19 +3,19 @@ local inspect = require "inspect"
local log = require "logger"
local properties = require "properties"
local scraper = require "scraper"
-local generator = require "generator"
+-- local generator = require "generator"
log:info(
inspect { message = "Start extraction", base_url = properties.base_url }
)
--- local index = crawler.fetch(properties.base_url .. properties.index_uri)
--- local modules = scraper.get_modules_from_index(
--- index,
--- properties.ignored_modules
--- )
+local index = crawler.fetch(properties.base_url .. properties.index_uri)
--- log:info(inspect { modules_found = #modules })
+-- local modules =
+-- scraper.get_modules_from_index(index, properties.ignored_modules)
+local module_infos = scraper.module_info_list.get_modules_from_index(index)
+
+log:info(inspect { modules_found = #module_infos })
-- for i = 1, 1 do -- #modules do
-- local m = modules[i]
@@ -25,12 +25,15 @@ log:info(
-- log:info(inspect { items })
-- end
-local page =
- crawler.fetch(properties.base_url .. "/widgets/wibox.widget.imagebox.html")
-local items = scraper.get_doc_from_page(page)
+local html =
+ crawler.fetch(properties.base_url .. "/widgets/wibox.widget.textbox.html")
+local module_doc = scraper.module_doc.get_doc_from_page(html)
+log:info(inspect { module_doc = module_doc })
+
+-- local items = scraper.get_doc_from_page(page)
-- log:info(inspect { items })
-generator.write(
- generator.generate_teal(items),
- properties.out_directory .. "/test.tl"
-)
+-- generator.write(
+-- generator.generate_teal(items),
+-- properties.out_directory .. "/test.tl"
+-- )
diff --git a/src/awesomewm.d.tl/scraper/init.lua b/src/awesomewm.d.tl/scraper/init.lua
index 6aa1cb2..77155e4 100644
--- a/src/awesomewm.d.tl/scraper/init.lua
+++ b/src/awesomewm.d.tl/scraper/init.lua
@@ -1,212 +1,5 @@
-local htmlparser = require "htmlparser"
-local log = require "logger"
-local utils = require "utils"
-
-local scraper = {}
-
-function scraper.get_modules_from_index(html, ignored)
- local document = htmlparser.parse(html)
-
- local modules = utils.map(document "#navigation ul > li a", function(node)
- return {
- name = utils.sanitize_string(node:getcontent()),
- uri = node.attributes.href,
- }
- end)
-
- local filtered_modules = utils.filter(modules, function(module)
- return not utils.has_item(ignored, module.name)
- end)
-
- return filtered_modules
-end
-
-local function extract_first_node_or_fail(node, selector)
- local extracted = node(selector)[1]
-
- if not extracted then
- log:error {
- message = "Can't find `" .. selector .. "` element!",
- node = node:gettext(),
- }
- error "extract_first_node_or_fail"
- end
-
- return extracted
-end
-
-local function extract_first_node(node, selector)
- local extracted = node(selector)[1]
-
- if not extracted then
- return nil
- end
-
- return extracted
-end
-
-local function extract_inner_floating_text(node)
- local html = node:getcontent()
-
- -- Remove inner tags from the html
- for _, n in ipairs(node.nodes) do
- html = utils.replace(html, n:gettext(), "")
- end
-
- return utils.sanitize_string(html)
-end
-
-local function extract_item_content_parameter_list(content)
- for i, n in ipairs(content.nodes) do
- -- The parameters
is the next element after one of these
- if
- n.name == "h3"
- and utils.has_item(
- { "Parameters:", "Type constraints:", "Arguments" },
- utils.sanitize_string(n:getcontent())
- )
- then
- return content.nodes[i + 1]
- end
- end
-
- return nil
-end
-
-local function process_item_header_type(node)
- local types_node = extract_first_node(node, "span.summary_type")
-
- if not types_node then
- return nil
- end
-
- local types_string = string.match(types_node:getcontent(), "%((.-)%)")
- or types_node:getcontent()
- types_string = utils.replace(types_string, "or", ",")
- local splitted = string.gmatch(types_string, "([^,]+)")
-
- local types = {}
- for t in splitted do
- table.insert(types, utils.sanitize_string(t))
- end
-
- return types
-end
-
-local function process_item_header(node)
- local name_node = extract_first_node_or_fail(node, "strong")
-
- -- Remove starting ":" character on method name
- local name = extract_inner_floating_text(name_node):gsub("^:", "")
- local type = process_item_header_type(node)
-
- local item_header = {
- name = name,
- type = type,
- }
-
- return item_header
-end
-
-local function process_item_content_parameters(node)
- local parameter = {}
-
- local name = extract_first_node_or_fail(node, "span.parameter")
- parameter.name = utils.sanitize_string(name:getcontent())
-
- local nested_parameters_list = extract_first_node(node, "ul")
- if nested_parameters_list then
- parameter.type = "table"
- parameter.nested_parameters = utils.map(
- nested_parameters_list "* > li",
- process_item_content_parameters
- )
-
- return parameter
- end
-
- local types_list = utils.map(node ".type", function(type)
- return utils.sanitize_string(type:getcontent())
- end)
- if #types_list > 0 then
- parameter.type = types_list
- end
-
- return parameter
-end
-
-local function process_item_content(content)
- local parameters_list = extract_item_content_parameter_list(content)
- local parameters = parameters_list
- and utils.map(
- parameters_list "* > li",
- process_item_content_parameters
- )
- or {}
-
- local item_content = {
- parameters = #parameters > 0 and parameters or nil,
- }
-
- return item_content
-end
-
-local function process_section_titles(document)
- return utils.map(document "h2.section-header", extract_inner_floating_text)
-end
-
-local function process_section_items(item)
- local headers = utils.map(item "dt", process_item_header)
- local contents = utils.map(item "dd", process_item_content)
-
- if #headers ~= #contents then
- log:error {
- message = "extract_item_content failure: headers and contents don't have the same size",
- headers = #headers,
- contents = #contents,
- }
- error "extract_item_content"
- end
-
- local item_contents = utils.map(headers, function(header, i)
- return {
- name = header.name,
- type = header.type,
- parameters = contents[i].parameters,
- }
- end)
-
- return item_contents
-end
-
-local function process_section_contents(document)
- local section_items = document "dl.function"
-
- local section_contents = utils.map(section_items, process_section_items)
-
- return section_contents
-end
-
-function scraper.get_doc_from_page(html)
- local document = htmlparser.parse(html, 9999)
-
- local sections_titles = process_section_titles(document)
- local section_contents = process_section_contents(document)
-
- if #sections_titles ~= #section_contents then
- log:error {
- message = "get_items_from_page failure: section_titles and section_contents don't have the same size",
- sections_titles = #sections_titles,
- func = #section_contents,
- }
- error "get_items_from_page"
- end
-
- local doc = utils.map(sections_titles, function(title, i)
- return { section = title, items = section_contents[i] }
- end)
-
- return doc
-end
-
-return scraper
+return {
+ module_doc = require "scraper.module_doc",
+ module_info_list = require "scraper.module_info_list",
+ utils = require "scraper.utils",
+}