From b12721e1fdcc2a86b126883d147620e28dd3c5e0 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 29 Jan 2023 19:19:20 +0100 Subject: [PATCH 01/39] feat(types): introduce new data structures --- src/awesomewm.d.tl/ast.tl | 63 ++++++++++++++++++++++++++++++ src/awesomewm.d.tl/dag.tl | 27 +++++++++++++ src/awesomewm.d.tl/types/Dag.d.tl | 8 ++++ src/awesomewm.d.tl/types/Node.d.tl | 25 ++++++++++++ 4 files changed, 123 insertions(+) create mode 100644 src/awesomewm.d.tl/ast.tl create mode 100644 src/awesomewm.d.tl/dag.tl create mode 100644 src/awesomewm.d.tl/types/Dag.d.tl create mode 100644 src/awesomewm.d.tl/types/Node.d.tl diff --git a/src/awesomewm.d.tl/ast.tl b/src/awesomewm.d.tl/ast.tl new file mode 100644 index 0000000..b4a469f --- /dev/null +++ b/src/awesomewm.d.tl/ast.tl @@ -0,0 +1,63 @@ +local type Node = require("types.Node") + +local basic_nodes : { Node.Token : function(name: string): Node } = { + module = function(name: string): Node + return { + token = "module", + name = name, + children = {}, + } + end, + record = function(name: string): Node + return { + token = "record", + name = name, + children = {}, + } + end, + enum = function(name: string): Node + return { + token = "enum", + name = name, + children = {}, + } + end, + identifier = function(name: string): Node + return { + token = "identifier", + name = name, + } + end, + variable = function(name: string): Node + return { + token = "variable", + name = name, + types = {}, + } + end, + ["function"] = function(name: string): Node + return { + token = "function", + name = name, + parameters = {}, + return_types = {}, + } + end, + metamethod = function(name: string): Node + return { + token = "metamethod", + name = name, + parameters = {}, + return_types = {}, + } + end, +} + +local function create_node(token: Node.Token, name: string): Node + local node = basic_nodes[token](name) + return node +end + +return { + create_node = create_node, +} diff --git a/src/awesomewm.d.tl/dag.tl b/src/awesomewm.d.tl/dag.tl new file mode 100644 index 0000000..b2ff5bf --- /dev/null +++ b/src/awesomewm.d.tl/dag.tl @@ -0,0 +1,27 @@ +local type Dag = require("types.Dag") +local type Node = require("types.Node") + +local function init(): Dag + local dag : Dag = { + nodes_by_module_name = {}, + global_nodes = {}, + } + return dag +end + +local function insert(dag: Dag, module_name: string, node: Node) + if not dag.nodes_by_module_name[module_name] then + dag.nodes_by_module_name[module_name] = {} + end + table.insert(dag.nodes_by_module_name[module_name], node) +end + +local function insert_global(dag: Dag, node: Node) + table.insert(dag.global_nodes, node) +end + +return { + init = init, + insert = insert, + insert_global = insert_global, +} diff --git a/src/awesomewm.d.tl/types/Dag.d.tl b/src/awesomewm.d.tl/types/Dag.d.tl new file mode 100644 index 0000000..c6d38bc --- /dev/null +++ b/src/awesomewm.d.tl/types/Dag.d.tl @@ -0,0 +1,8 @@ +local type Node = require("redo.ast.Node") + +local record Dag + nodes_by_module_name: { string : { Node } } + global_nodes: { Node } +end + +return Dag diff --git a/src/awesomewm.d.tl/types/Node.d.tl b/src/awesomewm.d.tl/types/Node.d.tl new file mode 100644 index 0000000..b0440aa --- /dev/null +++ b/src/awesomewm.d.tl/types/Node.d.tl @@ -0,0 +1,25 @@ +local record Node + enum Token + "module" -- file root node, it is always a record and the generated .d.tl file returns it + "record" + "enum" + "identifier" -- blank token with only a name (used for enum values) + "variable" + "function" + "metamethod" + end + token: Token + name: string + + -- for "module", "record", "enum" + children: { Node } + + -- for "variable" + types: { string } + + -- for "function" and "metamethod" + parameters: { Node } + return_types: { string } +end + +return Node From ab28bf8717e7c20e3ff7ea33cda46939e8d4ebc1 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 29 Jan 2023 19:21:45 +0100 Subject: [PATCH 02/39] feat(scraper): move "Object properties" to AST --- src/awesomewm.d.tl/scraper/module_doc.tl | 375 ++++++++++++----------- src/awesomewm.d.tl/scraper/utils.tl | 40 +++ src/awesomewm.d.tl/utils.tl | 10 + 3 files changed, 251 insertions(+), 174 deletions(-) diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index 6c7bd66..ea867c4 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -1,14 +1,10 @@ -local Function_Info = require "entity.Function_Info" -local List = require "pl.List" +local ast = require("ast") +local type Node = require("types.Node") local logger = require "logger" -local Map = require "pl.Map" -local Module_Doc = require "entity.Module_Doc" local scan = require "web_sanitize.query.scan_html" local scraper_utils = require "scraper.utils" local stringx = require "pl.stringx" -local Type_Info = require "entity.Type_Info" local utils = require "utils" -local Variable_Info = require "entity.Variable_Info" local log = logger.log("scraper") @@ -16,94 +12,93 @@ local function extract_node_text(node: scan.HTMLNode): string return utils.sanitize_string(node:inner_text()) end -local function parse_parameter_types(parameter_type: string): List +local function parse_parameter_types(parameter_type: string): { string } if parameter_type == "" then - local type_info: Type_Info.Type_Info = Type_Info("any") - return List({ type_info }) + return { "any" } end - return stringx.split(parameter_type, " or "):map( - function(type_name: string): Type_Info.Type_Info - return Type_Info(utils.sanitize_string(type_name)) - end - ) + local types = {} + for t in stringx.split(parameter_type, " or "):iter() do + table.insert(types, t) + end + return types end local function extract_item_name(item_name_node: scan.HTMLNode): string return item_name_node and ((item_name_node.attr.name as string):gsub("^.*[%.:]", "")) end -local function extract_function_parameter_Parameters(tr_node: scan.HTMLNode): { Variable_Info.Variable_Info } - local query_selectors = { - name = "span.parameter", - types = "span.types" - } +-- local function extract_function_parameter_Parameters(tr_node: scan.HTMLNode): { Variable_Info.Variable_Info } +-- local query_selectors = { +-- name = "span.parameter", +-- types = "span.types" +-- } - return scraper_utils.scrape_tuples( - tr_node:outer_html(), - { query_selectors.name, query_selectors.types }, - function(nodes: { string : scan.HTMLNode | nil }): Variable_Info.Variable_Info - return Variable_Info( - extract_node_text(nodes[query_selectors.name] as scan.HTMLNode), - parse_parameter_types(extract_node_text(nodes[query_selectors.types] as scan.HTMLNode)) - ) - end) -end +-- return scraper_utils.scrape_tuples( +-- tr_node:outer_html(), +-- { query_selectors.name, query_selectors.types }, +-- function(nodes: { string : scan.HTMLNode | nil }): Variable_Info.Variable_Info +-- return Variable_Info( +-- extract_node_text(nodes[query_selectors.name] as scan.HTMLNode), +-- parse_parameter_types(extract_node_text(nodes[query_selectors.types] as scan.HTMLNode)) +-- ) +-- end) +-- end -local function extract_function_parameters(function_parameters_node: scan.HTMLNode): { Variable_Info.Variable_Info } - local current_record_parameter: Type_Info.Type_Info | nil = nil +-- local function extract_function_parameters(function_parameters_node: scan.HTMLNode): { Variable_Info.Variable_Info } +-- local current_record_parameter: Type_Info.Type_Info | nil = nil - return scraper_utils.scrape( - function_parameters_node:outer_html(), - "tr", - function(line_node: scan.HTMLNode): Variable_Info.Variable_Info - local parameters = extract_function_parameter_Parameters(line_node) - if #parameters == 0 then - return nil - elseif #parameters ~= 1 then - log:error(logger.message_with_metadata("Expected 1 parameter by node", - { len = #parameters, line_node = line_node, parameters = parameters })) - error("Expected 1 parameter by node") - end - local name, types = parameters[1].name, parameters[1].types +-- return scraper_utils.scrape( +-- function_parameters_node:outer_html(), +-- "tr", +-- function(line_node: scan.HTMLNode): Variable_Info.Variable_Info +-- local parameters = extract_function_parameter_Parameters(line_node) +-- if #parameters == 0 then +-- return nil +-- elseif #parameters ~= 1 then +-- log:error(logger.message_with_metadata("Expected 1 parameter by node", +-- { len = #parameters, line_node = line_node, parameters = parameters })) +-- error("Expected 1 parameter by node") +-- end +-- local name, types = parameters[1].name, parameters[1].types - if line_node.attr ~= nil and line_node.attr.class == "see_also_sublist" and current_record_parameter then - local record_parameter = current_record_parameter as Type_Info.Type_Info - if not record_parameter.record_entries then - record_parameter.record_entries = Map() - end +-- if line_node.attr ~= nil and line_node.attr.class == "see_also_sublist" and current_record_parameter then +-- local record_parameter = current_record_parameter as Type_Info.Type_Info +-- if not record_parameter.record_entries then +-- record_parameter.record_entries = Map() +-- end - (record_parameter.record_entries as Map>):set(name, types) +-- (record_parameter.record_entries as Map>):set(name, types) - return nil - end +-- return nil +-- end - if #types == 1 and types[1].name == "table" then - local record_name = utils.capitalize(name) - current_record_parameter = Type_Info(record_name) - return Variable_Info( - name, - List({ current_record_parameter }) - ) - end +-- if #types == 1 and types[1].name == "table" then +-- local record_name = utils.capitalize(name) +-- current_record_parameter = Type_Info(record_name) +-- return Variable_Info( +-- name, +-- List({ current_record_parameter }) +-- ) +-- end - return Variable_Info(name, types) - end) -end +-- return Variable_Info(name, types) +-- end) +-- end -local function extract_function_return_types(function_return_types_node: scan.HTMLNode): List - if not function_return_types_node then - return {} - end +-- local function extract_function_return_types(function_return_types_node: scan.HTMLNode): List +-- if not function_return_types_node then +-- return {} +-- end - local selector = "span.types .type" - local html = function_return_types_node:outer_html() +-- local selector = "span.types .type" +-- local html = function_return_types_node:outer_html() - return List(scraper_utils.scrape(html, selector, extract_node_text)):map( - function(type_name: string): Type_Info.Type_Info - return Type_Info(type_name) - end) -end +-- return List(scraper_utils.scrape(html, selector, extract_node_text)):map( +-- function(type_name: string): Type_Info.Type_Info +-- return Type_Info(type_name) +-- end) +-- end local function extract_property_constraints(property_constraint_node: scan.HTMLNode): { string } return scraper_utils.scrape( @@ -113,147 +108,179 @@ local function extract_property_constraints(property_constraint_node: scan.HTMLN ) end -local function extract_section_functions(dl: string): { Function_Info.Function_Info } - local query_selectors = { - header = "dt", - name = "a", - body = "dd", - parameters = "table", - return_types = "ol", - } +-- local function extract_section_functions(dl: string): { Function_Info.Function_Info } +-- local query_selectors = { +-- header = "dt", +-- name = "a", +-- body = "dd", +-- parameters = "table", +-- return_types = "ol", +-- } - return scraper_utils.scrape_tuples( - dl, - { query_selectors.header, query_selectors.body }, - function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Function_Info - if not nodes[query_selectors.header] or not nodes[query_selectors.body] then - log:warn( - logger.message_with_metadata( - "Missing header or body", - { nodes = nodes } - ) - ) - error("Missing header or body") - end - local header = nodes[query_selectors.header] as scan.HTMLNode - local body = nodes[query_selectors.body] as scan.HTMLNode - local body_elements = scraper_utils.extract_nodes( - body:outer_html(), - { query_selectors.parameters, query_selectors.return_types } - ) - return Function_Info( - scraper_utils.scrape( - header:outer_html(), - query_selectors.name, - extract_item_name - )[1], - #body_elements:get(query_selectors.parameters) ~= 0 and - List(extract_function_parameters(body_elements:get(query_selectors.parameters)[1])) or - (List() as List), - #body_elements:get(query_selectors.return_types) ~= 0 and - extract_function_return_types(body_elements:get(query_selectors.return_types)[1]) or - (List() as List) - ) - end - ) -end +-- return scraper_utils.scrape_tuples( +-- dl, +-- { query_selectors.header, query_selectors.body }, +-- function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Function_Info +-- if not nodes[query_selectors.header] or not nodes[query_selectors.body] then +-- log:warn( +-- logger.message_with_metadata( +-- "Missing header or body", +-- { nodes = nodes } +-- ) +-- ) +-- error("Missing header or body") +-- end +-- local header = nodes[query_selectors.header] as scan.HTMLNode +-- local body = nodes[query_selectors.body] as scan.HTMLNode +-- local body_elements = scraper_utils.extract_nodes( +-- body:outer_html(), +-- { query_selectors.parameters, query_selectors.return_types } +-- ) +-- return Function_Info( +-- scraper_utils.scrape( +-- header:outer_html(), +-- query_selectors.name, +-- extract_item_name +-- )[1], +-- #body_elements:get(query_selectors.parameters) ~= 0 and +-- List(extract_function_parameters(body_elements:get(query_selectors.parameters)[1])) or +-- (List() as List), +-- #body_elements:get(query_selectors.return_types) ~= 0 and +-- extract_function_return_types(body_elements:get(query_selectors.return_types)[1]) or +-- (List() as List) +-- ) +-- end +-- ) +-- end -local function extract_section_variables(dl: string): { Variable_Info.Variable_Info } - local query_selectors = { +local function extract_section_variables(dl: string): { Node }, { string } + local query_selectors : { string : string } = { variable_name = "dt a", variable_summary_type = "dt span.summary_type", variable_property_constraint = "dd span.property_type", } - return scraper_utils.scrape_tuples( + local variables = {} + local signals = {} + + for nodes in scraper_utils.iter_tuples( dl, - { query_selectors.variable_name, query_selectors.variable_summary_type, query_selectors.variable_property_constraint }, - function(nodes: { string : scan.HTMLNode | nil }): Variable_Info.Variable_Info - local variable_info = Variable_Info() + utils.values(query_selectors) + ) do + local node = ast.create_node("variable", extract_item_name(nodes[query_selectors.variable_name])) + node.types = parse_parameter_types(extract_node_text(nodes[query_selectors.variable_summary_type])) - variable_info.name = extract_item_name(nodes[query_selectors.variable_name]) - variable_info.types = parse_parameter_types(extract_node_text(nodes[query_selectors.variable_summary_type])) - - if #variable_info.types == 1 and variable_info.types[1].name == "string" then - log:debug("extract variable string with constraints, this is an enum") - variable_info.constraints = List(extract_property_constraints(nodes[query_selectors.variable_property_constraint])):map( - function(constraint: string): string - return (constraint:gsub(""", "")) - end + if #node.types == 1 and node.types[1] == "string" then + log:debug("extract variable string with constraints, this is an enum") + local type_enum = ast.create_node("enum", utils.capitalize(node.name)) + for _, constraint in ipairs(extract_property_constraints(nodes[query_selectors.variable_property_constraint])) do + table.insert( + type_enum.children, + ast.create_node("identifier", (constraint:gsub(""", ""))) ) end - - return variable_info + table.insert(variables, type_enum) + node.types = { type_enum.name } end - ) + + table.insert(variables, node) + table.insert(signals, string.format("property::%s", node.name)) -- TODO : actually scrape the signals from the doc + end + + return variables, signals end -local function extract_section_signal(dl: string): { string } - local selector = "dt strong" +-- local function extract_section_signal(dl: string): { string } +-- local selector = "dt strong" - return scraper_utils.scrape(dl, selector, extract_node_text) -end +-- return scraper_utils.scrape(dl, selector, extract_node_text) +-- end local enum Section - "Constructors" - "Static module functions" + -- "Constructors" + -- "Static module functions" "Object properties" - "Object methods" - "Signals" + -- "Object methods" + -- "Signals" end -local section_scrapers: { Section : function(html: string, module_doc: Module_Doc.Module_Doc) } = { - ["Constructors"] = function(html: string, module_doc: Module_Doc.Module_Doc) - module_doc.constructors = List(extract_section_functions(html)) - end, - ["Static module functions"] = function(html: string, module_doc: Module_Doc.Module_Doc) - module_doc.static_functions = List(extract_section_functions(html)) - end, - ["Object properties"] = function(html: string, module_doc: Module_Doc.Module_Doc) - module_doc.properties = List(extract_section_variables(html)) - end, - ["Object methods"] = function(html: string, module_doc: Module_Doc.Module_Doc) - local self_parameter = Variable_Info("self", List({ Type_Info(module_doc.record_name) })) - module_doc.methods = List(extract_section_functions(html)):map( - function(method: Function_Info.Function_Info): Function_Info.Function_Info - method.parameters:insert(1, self_parameter) - return method - end - ) - end, - ["Signals"] = function(html: string, module_doc: Module_Doc.Module_Doc) - module_doc.signals = List(extract_section_signal(html)) +-- returns +-- - Nodes that should be added to the module +-- - Nodes that should be added to the global scope +-- - Strings that should be added to the record Signals +local section_scrapers : { Section : function(html: string): { Node }, { Node }, { string } } = { + -- ["Constructors"] = function(html: string, module_doc: Module_Doc.Module_Doc) + -- module_doc.constructors = List(extract_section_functions(html)) + -- end, + -- ["Static module functions"] = function(html: string, module_doc: Module_Doc.Module_Doc) + -- module_doc.static_functions = List(extract_section_functions(html)) + -- end, + ["Object properties"] = function(html: string): { Node }, { Node }, { string } + local properties, signals = extract_section_variables(html) + return properties, {}, signals end, + -- ["Object methods"] = function(html: string, module_doc: Module_Doc.Module_Doc) + -- local self_parameter = Variable_Info("self", List({ Type_Info(module_doc.record_name) })) + -- module_doc.methods = List(extract_section_functions(html)):map( + -- function(method: Function_Info.Function_Info): Function_Info.Function_Info + -- method.parameters:insert(1, self_parameter) + -- return method + -- end + -- ) + -- end, + -- ["Signals"] = function(html: string, module_doc: Module_Doc.Module_Doc) + -- module_doc.signals = List(extract_section_signal(html)) + -- end, } +-- local function extract_node_module_name(node: Node): string +-- return (node.name:gsub("(.*)[%.:].+$", "%1")) +-- end + local module = {} -function module.get_doc_from_page(html: string, module_name: string): Module_Doc.Module_Doc - local nodes = scraper_utils.extract_nodes(html, { +function module.get_doc_from_page(html: string, module_name: string): Node, { Node } + local html_nodes = scraper_utils.extract_nodes(html, { "h2.section-header", "dl.function", }) - if #nodes:get "h2.section-header" ~= #nodes:get "dl.function" then + if #html_nodes:get "h2.section-header" ~= #html_nodes:get "dl.function" then error "The list aren't the same size!" end - local module_doc = Module_Doc() - module_doc.record_name = utils.capitalize((module_name:gsub(".*%.", ""))) + local record_name = utils.capitalize((module_name:gsub(".*%.", ""))) + local module_root = ast.create_node("module", record_name) + local other_nodes : { Node } = {} - for i = 1, #nodes:get("h2.section-header") do - local h2 = nodes:get("h2.section-header")[i] + local module_signals_node = ast.create_node("enum", "Signal") + table.insert(module_root.children, module_signals_node) + + for i = 1, #html_nodes:get("h2.section-header") do + local h2 = html_nodes:get("h2.section-header")[i] local section_name = utils.sanitize_string(h2:inner_text()) as Section -- promote to Section, we then test if the section_name is in the table - local dl_html = nodes:get("dl.function")[i]:outer_html() + local dl_html = html_nodes:get("dl.function")[i]:outer_html() if section_scrapers[section_name] then - section_scrapers[section_name](dl_html, module_doc) + local module_nodes, global_nodes, signals_name = section_scrapers[section_name](dl_html) + for _, node in ipairs(module_nodes) do + table.insert(module_root.children, node) + end + for _, node in ipairs(global_nodes) do + table.insert(other_nodes, node) + end + for _, signal_name in ipairs(signals_name) do + table.insert( + module_signals_node.children, + ast.create_node("identifier", signal_name) + ) + end else log:warn("Section scraper not implemented: " .. section_name) end end - return module_doc + return module_root, other_nodes end return module diff --git a/src/awesomewm.d.tl/scraper/utils.tl b/src/awesomewm.d.tl/scraper/utils.tl index 26ff900..33032ca 100644 --- a/src/awesomewm.d.tl/scraper/utils.tl +++ b/src/awesomewm.d.tl/scraper/utils.tl @@ -72,4 +72,44 @@ function scraper_utils.scrape_tuples(html: string, query_selectors: { string return ret end +function scraper_utils.iter_tuples(html: string, query_selectors: { string }): function(): { string : scan.HTMLNode } + local siblings: { string : { scan.HTMLNode } } = {} + for _, query_selector in ipairs(query_selectors) do + siblings[query_selector] = {} + end + + scanner.scan_html( + html, + function(stack: scan.NodeStack) + for _, query_selector in ipairs(query_selectors) do + if stack:is(query_selector) then + table.insert(siblings[query_selector], stack:current()) + end + end + end + ) + + local siblings_count = #siblings[query_selectors[1]] + for _, query_selector in ipairs(query_selectors) do + if #siblings[query_selector] ~= siblings_count then + error("Query selectors do not have the same number of siblings") + end + end + + local i = 0 + return function(): { string : scan.HTMLNode } + i = i + 1 + if i > siblings_count then + return nil + end + + local node_list: { string : scan.HTMLNode } = {} + for _, query_selector in ipairs(query_selectors) do + node_list[query_selector] = siblings[query_selector][i] + end + + return node_list + end +end + return scraper_utils diff --git a/src/awesomewm.d.tl/utils.tl b/src/awesomewm.d.tl/utils.tl index 828070e..824b9dc 100644 --- a/src/awesomewm.d.tl/utils.tl +++ b/src/awesomewm.d.tl/utils.tl @@ -35,6 +35,16 @@ function utils.map(list: { T }, iteratee: function(value: T, position: int return mapped end +function utils.values(t: table): { T } + local values: { T } = {} + + for _, v in pairs(t) do + table.insert(values, v as T) + end + + return values +end + function utils.sanitize_string(s: string): string return (stringx.strip(web_sanitize.extract_text(s))) end From 09181259e12cea1effe7a429a462c1fb5cd2cf14 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 29 Jan 2023 19:22:19 +0100 Subject: [PATCH 03/39] run: some play with the entrypoint --- src/awesomewm.d.tl/init.tl | 152 ++++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 71 deletions(-) diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index b508aab..5f228eb 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -65,78 +65,88 @@ local function modules_tree(modules: List): Map = List() +-- for module in global_module_infos:iter() do +-- if module.name:gmatch(".*%sand%s.*") then +-- do_one_file( +-- property.base_url .. "/" .. module.uri, +-- module.name, +-- property.out_directory .. "/" .. module.name:gsub(".*%sand%s", ""):gsub("%.", "/") .. ".d.tl" +-- ) +-- end + +-- local html = crawler.fetch(property.base_url .. "/" .. module.uri) +-- local module_doc = scraper.module_doc.get_doc_from_page(html, (module.name:gsub("%sand%s.*", ""))) +-- module_doc:fixup() +-- module_doc.record_name = utils.lowercase(module_doc.record_name) +-- global_env_def:append(module_doc) +-- end +-- filesystem.file_writer.write( +-- generator.global_env_def.generate_teal(global_env_def), +-- property.out_directory .. "/global_env.d.tl" +-- ) + +-- for module, children in tree:iter() do +-- -- TODO : this map should be coupled with the all_module_infos list +-- local requires: Map = Map() +-- for child in children:iter() do +-- local name = child:gmatch(".*%.(.*)$")() +-- requires:set(name, child) +-- end +-- filesystem.file_writer.write( +-- generator.module_init_definition.generate_teal(requires), +-- property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl" +-- ) +-- end + +local module_ast, other_nodes = scraper.module_doc.get_doc_from_page( + crawler.fetch(property.base_url .. "/widgets/wibox.widget.imagebox.html"), + "awful.tag" ) -for module in module_infos:iter() do - do_one_file( - property.base_url .. "/" .. module.uri, - module.name, - property.out_directory .. "/" .. module.name:gsub("%.", "/") .. ".d.tl" - ) -end - -local global_env_def: List = List() -for module in global_module_infos:iter() do - if module.name:gmatch(".*%sand%s.*") then - do_one_file( - property.base_url .. "/" .. module.uri, - module.name, - property.out_directory .. "/" .. module.name:gsub(".*%sand%s", ""):gsub("%.", "/") .. ".d.tl" - ) - end - - local html = crawler.fetch(property.base_url .. "/" .. module.uri) - local module_doc = scraper.module_doc.get_doc_from_page(html, (module.name:gsub("%sand%s.*", ""))) - module_doc:fixup() - module_doc.record_name = utils.lowercase(module_doc.record_name) - global_env_def:append(module_doc) -end -filesystem.file_writer.write( - generator.global_env_def.generate_teal(global_env_def), - property.out_directory .. "/global_env.d.tl" -) - -for module, children in tree:iter() do - -- TODO : this map should be coupled with the all_module_infos list - local requires: Map = Map() - for child in children:iter() do - local name = child:gmatch(".*%.(.*)$")() - requires:set(name, child) - end - filesystem.file_writer.write( - generator.module_init_definition.generate_teal(requires), - property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl" - ) -end +log:info(logger.message_with_metadata("Finished", { + module_ast = module_ast, + other_nodes = other_nodes, +})) From eaf31b2f166e1f23bd5ff83d9475fdd8241eab6c Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 4 Feb 2023 00:40:43 +0100 Subject: [PATCH 04/39] feat(scraper): move "Signals" to AST --- src/awesomewm.d.tl/scraper/module_doc.tl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index ea867c4..230e82a 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -190,18 +190,18 @@ local function extract_section_variables(dl: string): { Node }, { string } return variables, signals end --- local function extract_section_signal(dl: string): { string } --- local selector = "dt strong" +local function extract_section_signal(dl: string): { string } + local selector = "dt strong" --- return scraper_utils.scrape(dl, selector, extract_node_text) --- end + return scraper_utils.scrape(dl, selector, extract_node_text) +end local enum Section -- "Constructors" -- "Static module functions" "Object properties" -- "Object methods" - -- "Signals" + "Signals" end -- returns @@ -228,9 +228,10 @@ local section_scrapers : { Section : function(html: string): { Node }, { -- end -- ) -- end, - -- ["Signals"] = function(html: string, module_doc: Module_Doc.Module_Doc) - -- module_doc.signals = List(extract_section_signal(html)) - -- end, + ["Signals"] = function(html: string): { Node }, { Node }, { string } + local signals = extract_section_signal(html) + return {}, {}, signals + end, } -- local function extract_node_module_name(node: Node): string From 3d2d009542e9ac498aab2e7a9e270d75929764f1 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 4 Feb 2023 00:41:20 +0100 Subject: [PATCH 05/39] chore: add cSpell wording --- .vscode/settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 80272e5..9825638 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -32,6 +32,7 @@ "mkdir", "rockspec", "setopt", + "stringx", "Stylua", "tablex", "tmpl", From f142f0e75069d7f9428161a1326ffdda76e87666 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 11 Feb 2023 18:32:57 +0100 Subject: [PATCH 06/39] fix(types): Dag requires path --- src/awesomewm.d.tl/types/Dag.d.tl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/awesomewm.d.tl/types/Dag.d.tl b/src/awesomewm.d.tl/types/Dag.d.tl index c6d38bc..481ffa2 100644 --- a/src/awesomewm.d.tl/types/Dag.d.tl +++ b/src/awesomewm.d.tl/types/Dag.d.tl @@ -1,4 +1,4 @@ -local type Node = require("redo.ast.Node") +local type Node = require("types.Node") local record Dag nodes_by_module_name: { string : { Node } } From 915673668246a42940840b3a58df55599eca2cc0 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Wed, 12 Apr 2023 00:19:53 +0200 Subject: [PATCH 07/39] feat(scraper): implement all `section_scrapers` --- .vscode/settings.json | 1 + src/awesomewm.d.tl/scraper/module_doc.tl | 222 ++++++++++------------- src/awesomewm.d.tl/scraper/utils.tl | 26 +-- 3 files changed, 101 insertions(+), 148 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9825638..3a1c509 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -34,6 +34,7 @@ "setopt", "stringx", "Stylua", + "sublist", "tablex", "tmpl", "wibox", diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index 230e82a..1bea452 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -28,77 +28,50 @@ local function extract_item_name(item_name_node: scan.HTMLNode): string return item_name_node and ((item_name_node.attr.name as string):gsub("^.*[%.:]", "")) end --- local function extract_function_parameter_Parameters(tr_node: scan.HTMLNode): { Variable_Info.Variable_Info } --- local query_selectors = { --- name = "span.parameter", --- types = "span.types" --- } +local function extract_function_parameters(table_html: string): { Node } + local current_record_parameter: Node = nil --- return scraper_utils.scrape_tuples( --- tr_node:outer_html(), --- { query_selectors.name, query_selectors.types }, --- function(nodes: { string : scan.HTMLNode | nil }): Variable_Info.Variable_Info --- return Variable_Info( --- extract_node_text(nodes[query_selectors.name] as scan.HTMLNode), --- parse_parameter_types(extract_node_text(nodes[query_selectors.types] as scan.HTMLNode)) --- ) --- end) --- end + return scraper_utils.scrape(table_html, "tr", function(tr: scan.HTMLNode): Node + local tr_html = tr:outer_html() + local name_node = scraper_utils.find(tr_html, "span.parameter")[1] + local types_node = scraper_utils.find(tr_html, "span.types")[1] + if not name_node or not types_node then + return nil + end --- local function extract_function_parameters(function_parameters_node: scan.HTMLNode): { Variable_Info.Variable_Info } --- local current_record_parameter: Type_Info.Type_Info | nil = nil + local name = extract_node_text(name_node) + local types = parse_parameter_types(extract_node_text(types_node)) --- return scraper_utils.scrape( --- function_parameters_node:outer_html(), --- "tr", --- function(line_node: scan.HTMLNode): Variable_Info.Variable_Info --- local parameters = extract_function_parameter_Parameters(line_node) --- if #parameters == 0 then --- return nil --- elseif #parameters ~= 1 then --- log:error(logger.message_with_metadata("Expected 1 parameter by node", --- { len = #parameters, line_node = line_node, parameters = parameters })) --- error("Expected 1 parameter by node") --- end --- local name, types = parameters[1].name, parameters[1].types + if tr.attr ~= nil and tr.attr.class == "see_also_sublist" and current_record_parameter then + local field = ast.create_node("variable", name) + field.types = types + table.insert(current_record_parameter.children, field) + return nil + end --- if line_node.attr ~= nil and line_node.attr.class == "see_also_sublist" and current_record_parameter then --- local record_parameter = current_record_parameter as Type_Info.Type_Info --- if not record_parameter.record_entries then --- record_parameter.record_entries = Map() --- end + -- We wrongly tried to convert a table to a record + if current_record_parameter then + current_record_parameter.token = "variable" + current_record_parameter.name = utils.lowercase(current_record_parameter.name) + current_record_parameter.types = { "table" } + current_record_parameter.children = nil + current_record_parameter = nil + end --- (record_parameter.record_entries as Map>):set(name, types) + if #types == 1 and types[1] == "table" then + current_record_parameter = ast.create_node("record", utils.capitalize(name)) + return current_record_parameter + end --- return nil --- end + local field = ast.create_node("variable", name) + field.types = types + return field + end) +end --- if #types == 1 and types[1].name == "table" then --- local record_name = utils.capitalize(name) --- current_record_parameter = Type_Info(record_name) --- return Variable_Info( --- name, --- List({ current_record_parameter }) --- ) --- end - --- return Variable_Info(name, types) --- end) --- end - --- local function extract_function_return_types(function_return_types_node: scan.HTMLNode): List --- if not function_return_types_node then --- return {} --- end - --- local selector = "span.types .type" --- local html = function_return_types_node:outer_html() - --- return List(scraper_utils.scrape(html, selector, extract_node_text)):map( --- function(type_name: string): Type_Info.Type_Info --- return Type_Info(type_name) --- end) --- end +local function extract_function_return_types(ol_html: string): { string } + return scraper_utils.scrape(ol_html, "span.types .type", extract_node_text) +end local function extract_property_constraints(property_constraint_node: scan.HTMLNode): { string } return scraper_utils.scrape( @@ -108,50 +81,40 @@ local function extract_property_constraints(property_constraint_node: scan.HTMLN ) end --- local function extract_section_functions(dl: string): { Function_Info.Function_Info } --- local query_selectors = { --- header = "dt", --- name = "a", --- body = "dd", --- parameters = "table", --- return_types = "ol", --- } +local function extract_section_functions(dl: string): { Node } + local list_query_selectors : { string : string } = { + function_name = "dt a", + body = "dd", + } --- return scraper_utils.scrape_tuples( --- dl, --- { query_selectors.header, query_selectors.body }, --- function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Function_Info --- if not nodes[query_selectors.header] or not nodes[query_selectors.body] then --- log:warn( --- logger.message_with_metadata( --- "Missing header or body", --- { nodes = nodes } --- ) --- ) --- error("Missing header or body") --- end --- local header = nodes[query_selectors.header] as scan.HTMLNode --- local body = nodes[query_selectors.body] as scan.HTMLNode --- local body_elements = scraper_utils.extract_nodes( --- body:outer_html(), --- { query_selectors.parameters, query_selectors.return_types } --- ) --- return Function_Info( --- scraper_utils.scrape( --- header:outer_html(), --- query_selectors.name, --- extract_item_name --- )[1], --- #body_elements:get(query_selectors.parameters) ~= 0 and --- List(extract_function_parameters(body_elements:get(query_selectors.parameters)[1])) or --- (List() as List), --- #body_elements:get(query_selectors.return_types) ~= 0 and --- extract_function_return_types(body_elements:get(query_selectors.return_types)[1]) or --- (List() as List) --- ) --- end --- ) --- end + local functions: { Node } = {} + + for nodes in scraper_utils.iter_tuples( + dl, + utils.values(list_query_selectors) + ) do + local function_node = ast.create_node( + "function", + extract_item_name(nodes[list_query_selectors.function_name]) + ) + + local body_html = nodes[list_query_selectors.body]:outer_html() + + local parameter_node = scraper_utils.find(body_html, "table")[1] + function_node.parameters = parameter_node and + extract_function_parameters(parameter_node:outer_html()) or + {} + + local return_node = scraper_utils.find(body_html, "ol")[1] + function_node.return_types = return_node and + extract_function_return_types(return_node:outer_html()) or + {} + + table.insert(functions, function_node) + end + + return functions +end local function extract_section_variables(dl: string): { Node }, { string } local query_selectors : { string : string } = { @@ -160,8 +123,8 @@ local function extract_section_variables(dl: string): { Node }, { string } variable_property_constraint = "dd span.property_type", } - local variables = {} - local signals = {} + local variables : { Node } = {} + local signals : { string } = {} for nodes in scraper_utils.iter_tuples( dl, @@ -197,10 +160,10 @@ local function extract_section_signal(dl: string): { string } end local enum Section - -- "Constructors" - -- "Static module functions" + "Constructors" + "Static module functions" "Object properties" - -- "Object methods" + "Object methods" "Signals" end @@ -208,26 +171,27 @@ end -- - Nodes that should be added to the module -- - Nodes that should be added to the global scope -- - Strings that should be added to the record Signals -local section_scrapers : { Section : function(html: string): { Node }, { Node }, { string } } = { - -- ["Constructors"] = function(html: string, module_doc: Module_Doc.Module_Doc) - -- module_doc.constructors = List(extract_section_functions(html)) - -- end, - -- ["Static module functions"] = function(html: string, module_doc: Module_Doc.Module_Doc) - -- module_doc.static_functions = List(extract_section_functions(html)) - -- end, +local section_scrapers : { Section : function(html: string, module_name: string): { Node }, { Node }, { string } } = { + ["Constructors"] = function(html: string): { Node }, { Node }, { string } + return extract_section_functions(html), {}, {} + end, + ["Static module functions"] = function(html: string): { Node }, { Node }, { string } + local static_functions = extract_section_functions(html) + return static_functions, {}, {} + end, ["Object properties"] = function(html: string): { Node }, { Node }, { string } local properties, signals = extract_section_variables(html) return properties, {}, signals end, - -- ["Object methods"] = function(html: string, module_doc: Module_Doc.Module_Doc) - -- local self_parameter = Variable_Info("self", List({ Type_Info(module_doc.record_name) })) - -- module_doc.methods = List(extract_section_functions(html)):map( - -- function(method: Function_Info.Function_Info): Function_Info.Function_Info - -- method.parameters:insert(1, self_parameter) - -- return method - -- end - -- ) - -- end, + ["Object methods"] = function(html: string, module_name: string): { Node }, { Node }, { string } + local methods = extract_section_functions(html) + for _, method in ipairs(methods) do + local self_parameter = ast.create_node("variable", "self") + self_parameter.types = { module_name } + table.insert(method.parameters, 1, self_parameter) + end + return methods, {}, {} + end, ["Signals"] = function(html: string): { Node }, { Node }, { string } local signals = extract_section_signal(html) return {}, {}, signals @@ -263,7 +227,7 @@ function module.get_doc_from_page(html: string, module_name: string): Node, { No local dl_html = html_nodes:get("dl.function")[i]:outer_html() if section_scrapers[section_name] then - local module_nodes, global_nodes, signals_name = section_scrapers[section_name](dl_html) + local module_nodes, global_nodes, signals_name = section_scrapers[section_name](dl_html, record_name) for _, node in ipairs(module_nodes) do table.insert(module_root.children, node) end diff --git a/src/awesomewm.d.tl/scraper/utils.tl b/src/awesomewm.d.tl/scraper/utils.tl index 33032ca..8454e03 100644 --- a/src/awesomewm.d.tl/scraper/utils.tl +++ b/src/awesomewm.d.tl/scraper/utils.tl @@ -48,28 +48,16 @@ function scraper_utils.extract_nodes(html: string, query_selectors: { string }): return siblings end -function scraper_utils.scrape_tuples(html: string, query_selectors: { string }, extract_callback: function(tuple: { string : scan.HTMLNode | nil }): T): { T } - local nodes = scraper_utils.extract_nodes(html, query_selectors) +function scraper_utils.find(html: string, query_selector: string): { scan.HTMLNode } + local nodes: { scan.HTMLNode } = {} - local ret: { T } = {} - - for i = 1, #nodes:get(query_selectors[1]) do - local node_list: { string : scan.HTMLNode | nil } = {} - tablex.foreach(query_selectors, function(query_selector: string) - node_list[query_selector] = nodes:get(query_selector)[i] or nil - end) - local success, info_or_error = pcall(extract_callback, node_list) - - if not success then - local error_message = info_or_error as string - log:error(logger.message_with_metadata("Extraction error", { error = error_message })) - else - local info = info_or_error as T - table.insert(ret, info) + scanner.scan_html(html, function(stack: scan.NodeStack) + if stack:is(query_selector) then + table.insert(nodes, stack:current()) end - end + end) - return ret + return nodes end function scraper_utils.iter_tuples(html: string, query_selectors: { string }): function(): { string : scan.HTMLNode } From 896c7f2f04ecb7fd7f1f8f0fce42b90c048a8bcc Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 16 Apr 2023 12:54:36 +0200 Subject: [PATCH 08/39] feat(scraper): function can populate `other_nodes` To allow `extract_section_functions` to populate `other_node`, we need to use the `module_name` and migrate the function name scrap method to use the actually displayed text instead of playing with the node `name` attribute. Because of this change, we also need to update `extract_section_variables`. Not a big deal. BTW, we resolved an issue where `string` variable can wrongly be detected as `enum`. --- .vscode/settings.json | 1 + src/awesomewm.d.tl/scraper/module_doc.tl | 57 +++++++++++++++--------- 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3a1c509..008bada 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -30,6 +30,7 @@ "luasec", "luasocket", "mkdir", + "modname", "rockspec", "setopt", "stringx", diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index 1bea452..972f5d8 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -24,8 +24,14 @@ local function parse_parameter_types(parameter_type: string): { string } return types end -local function extract_item_name(item_name_node: scan.HTMLNode): string - return item_name_node and ((item_name_node.attr.name as string):gsub("^.*[%.:]", "")) +local function extract_item_name(item_name_node: scan.HTMLNode): string, string | nil + if not item_name_node then + return + end + local module_name_node = scraper_utils.find(item_name_node:outer_html(), "span.function_modname")[1] + local module_name = module_name_node and module_name_node:inner_text():gsub("[%.:]$", "") + local name = item_name_node:inner_text():gsub("^.*[%.:](.+)%s*[%(%{].*[%)%}]", "%1") + return utils.sanitize_string(name), module_name and utils.sanitize_string(module_name) or nil end local function extract_function_parameters(table_html: string): { Node } @@ -81,21 +87,23 @@ local function extract_property_constraints(property_constraint_node: scan.HTMLN ) end -local function extract_section_functions(dl: string): { Node } +local function extract_section_functions(dl: string, module_name: string | nil): { Node }, { Node} local list_query_selectors : { string : string } = { - function_name = "dt a", + function_name = "dt strong", body = "dd", } - local functions: { Node } = {} + local functions : { Node } = {} + local other_functions : { Node } = {} for nodes in scraper_utils.iter_tuples( dl, utils.values(list_query_selectors) ) do + local function_name , function_module_name = extract_item_name(nodes[list_query_selectors.function_name]) local function_node = ast.create_node( "function", - extract_item_name(nodes[list_query_selectors.function_name]) + function_name ) local body_html = nodes[list_query_selectors.body]:outer_html() @@ -110,15 +118,20 @@ local function extract_section_functions(dl: string): { Node } extract_function_return_types(return_node:outer_html()) or {} - table.insert(functions, function_node) + if module_name and function_module_name and module_name ~= function_module_name then + function_node.name = function_module_name .. "." .. function_node.name + table.insert(other_functions, function_node) + else + table.insert(functions, function_node) + end end - return functions + return functions, other_functions end local function extract_section_variables(dl: string): { Node }, { string } local query_selectors : { string : string } = { - variable_name = "dt a", + variable_name = "dt strong", variable_summary_type = "dt span.summary_type", variable_property_constraint = "dd span.property_type", } @@ -130,11 +143,11 @@ local function extract_section_variables(dl: string): { Node }, { string } dl, utils.values(query_selectors) ) do - local node = ast.create_node("variable", extract_item_name(nodes[query_selectors.variable_name])) + local node = ast.create_node("variable", (extract_item_name(nodes[query_selectors.variable_name]))) node.types = parse_parameter_types(extract_node_text(nodes[query_selectors.variable_summary_type])) if #node.types == 1 and node.types[1] == "string" then - log:debug("extract variable string with constraints, this is an enum") + log:debug("extract variable string with constraints, this is an enum", { name = node.name }) local type_enum = ast.create_node("enum", utils.capitalize(node.name)) for _, constraint in ipairs(extract_property_constraints(nodes[query_selectors.variable_property_constraint])) do table.insert( @@ -142,8 +155,12 @@ local function extract_section_variables(dl: string): { Node }, { string } ast.create_node("identifier", (constraint:gsub(""", ""))) ) end - table.insert(variables, type_enum) - node.types = { type_enum.name } + if #type_enum.children == 0 then + log:debug("Enum has no children, get back to variable", { name = node.name }) + else + table.insert(variables, type_enum) + node.types = { type_enum.name } + end end table.insert(variables, node) @@ -171,23 +188,23 @@ end -- - Nodes that should be added to the module -- - Nodes that should be added to the global scope -- - Strings that should be added to the record Signals -local section_scrapers : { Section : function(html: string, module_name: string): { Node }, { Node }, { string } } = { +local section_scrapers : { Section : function(html: string, record_name: string, module_name: string): { Node }, { Node }, { string } } = { ["Constructors"] = function(html: string): { Node }, { Node }, { string } return extract_section_functions(html), {}, {} end, - ["Static module functions"] = function(html: string): { Node }, { Node }, { string } - local static_functions = extract_section_functions(html) - return static_functions, {}, {} + ["Static module functions"] = function(html: string, _: string, module_name: string): { Node }, { Node }, { string } + local static_functions, other_functions = extract_section_functions(html, module_name) + return static_functions, other_functions, {} end, ["Object properties"] = function(html: string): { Node }, { Node }, { string } local properties, signals = extract_section_variables(html) return properties, {}, signals end, - ["Object methods"] = function(html: string, module_name: string): { Node }, { Node }, { string } + ["Object methods"] = function(html: string, record_name: string): { Node }, { Node }, { string } local methods = extract_section_functions(html) for _, method in ipairs(methods) do local self_parameter = ast.create_node("variable", "self") - self_parameter.types = { module_name } + self_parameter.types = { record_name } table.insert(method.parameters, 1, self_parameter) end return methods, {}, {} @@ -227,7 +244,7 @@ function module.get_doc_from_page(html: string, module_name: string): Node, { No local dl_html = html_nodes:get("dl.function")[i]:outer_html() if section_scrapers[section_name] then - local module_nodes, global_nodes, signals_name = section_scrapers[section_name](dl_html, record_name) + local module_nodes, global_nodes, signals_name = section_scrapers[section_name](dl_html, record_name, module_name) for _, node in ipairs(module_nodes) do table.insert(module_root.children, node) end From 592e62d6fa04d8e1cae3e9e95c08cc377a9592ee Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 16 Apr 2023 22:15:47 +0200 Subject: [PATCH 09/39] feat(generator): base implem for `generate_teal` --- src/awesomewm.d.tl/ast.tl | 5 + .../generator/teal_type_definitions.tl | 152 ++++++++++++------ src/awesomewm.d.tl/init.tl | 7 +- 3 files changed, 117 insertions(+), 47 deletions(-) diff --git a/src/awesomewm.d.tl/ast.tl b/src/awesomewm.d.tl/ast.tl index b4a469f..0f8a608 100644 --- a/src/awesomewm.d.tl/ast.tl +++ b/src/awesomewm.d.tl/ast.tl @@ -58,6 +58,11 @@ local function create_node(token: Node.Token, name: string): Node return node end +local function iter_children(node: Node): function(): integer, Node + return ipairs(node.children) +end + return { create_node = create_node, + iter_children = iter_children, } diff --git a/src/awesomewm.d.tl/generator/teal_type_definitions.tl b/src/awesomewm.d.tl/generator/teal_type_definitions.tl index 0a1d31a..39393d1 100644 --- a/src/awesomewm.d.tl/generator/teal_type_definitions.tl +++ b/src/awesomewm.d.tl/generator/teal_type_definitions.tl @@ -1,56 +1,116 @@ -local Module_Doc = require "entity.Module_Doc" -local template = require "pl.template" -local utils = require "utils" -local snippets = require "generator.snippets" +local ast = require "ast" +local logger = require "logger" +local type Node = require "types.Node" --- The long therm goal is to have so many `snippets.render_*` functions that --- we can render the whole file with the smallest template possible. -local tmpl = [[ --- Auto generated file (Do not manually edit this file!) +local log = logger.log("scraper") -# if module.requires:len() ~= 0 then -$(snippets.render_requires(module.requires)) -# end -- /requires - -local record $(module.record_name) -# if #module.signals ~= 0 then -$(snippets.indent(snippets.render_enum("Signal", module.signals))) - -# end -- /signals -# if #module.methods ~= 0 then - -- Object methods -$(snippets.indent(snippets.render_record_functions(module.methods))) - -# end -- /methods -# if #module.properties ~= 0 then - -- Object properties -$(snippets.indent(snippets.render_record_properties(module.properties))) - -# end -- /properties -# if #module.constructors ~= 0 then - -- Constructors -$(snippets.indent(snippets.render_record_functions(module.constructors))) - -# end -- /constructors -# if #module.static_functions ~= 0 then - -- Static functions -$(snippets.indent(snippets.render_record_functions(module.static_functions))) -# end -- /static_functions +local function concatenate_strings(strings: { string }): string + local result = "" + for _, str in ipairs(strings) do + result = result .. str + end + return result end -return $(module.record_name) -]] +local function render_types(types: { string }): string + if not types or #types == 0 then + return "" + end -local module = {} + return ": " .. concatenate_strings(types) +end -function module.generate_teal(data: Module_Doc.Module_Doc): string - local tmpl_args = { - ipairs = ipairs, - module = data, - snippets = snippets, +local record Node_Generator_Function + on_node: function(node: Node): string + before_node: nil | function(node: Node): string + after_node: nil | function(node: Node): string +end + +-- pre-declare functions to prevent forward reference errors +local generate_teal: function(node: Node): string +local generate_children: function(node: Node): string + +local node_generator : { Node.Token : Node_Generator_Function } = { + ["module"] = { + before_node = function(node: Node): string + return "-- This file was auto-generated.\n\nlocal record " .. node.name .. "\n" + end, + on_node = function(node: Node): string + return generate_children(node) + end, + after_node = function(node: Node): string + return "end\n\nreturn " .. node.name + end, + }, + ["record"] = { + before_node = function(node: Node): string + return "local record " .. node.name + end, + on_node = function(node: Node): string + return generate_children(node) + end, + after_node = function(): string + return "end" + end, + }, + ["enum"] = { + before_node = function(node: Node): string + return "enum " .. node.name .. "\n" + end, + on_node = function(node: Node): string + return generate_children(node) + end, + after_node = function(): string + return "end" + end, + }, + ["identifier"] = { + on_node = function(node: Node): string + return "\"" .. node.name .. "\"" + end, + }, + ["variable"] = { + on_node = function(node: Node): string + return node.name .. render_types(node.types) + end, + }, + ["function"] = { + on_node = function(node: Node): string + local args = {} + for _, parameter in ipairs(node.parameters) do + table.insert(args, generate_teal(parameter)) + end + return node.name .. ": function(" .. table.concat(args, ", ") .. ")" .. render_types(node.return_types) + end, + }, + ["metamethod"] = { + on_node = function(): string + log:warn("Metamethods are not supported yet") + end, } - return utils.do_or_fail(template.substitute, tmpl, tmpl_args) +} +function generate_teal(node: Node): string + local generator = node_generator[node.token] + local generated = "" + if generator.before_node then + generated = (generator.before_node as function(node: Node): string)(node) + end + generated = generated .. generator.on_node(node) + if generator.after_node then + generated = generated .. (generator.after_node as function(node: Node): string)(node) + end + return generated end -return module +function generate_children(node: Node): string + local generated = "" + for _, child in ast.iter_children(node) do + generated = generated .. generate_teal(child) .. "\n" + end + return generated +end + +return { + generate_teal = generate_teal, +} diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index 5f228eb..605b341 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -142,7 +142,7 @@ end -- end local module_ast, other_nodes = scraper.module_doc.get_doc_from_page( - crawler.fetch(property.base_url .. "/widgets/wibox.widget.imagebox.html"), + crawler.fetch(property.base_url .. "/core_components/tag.html"), "awful.tag" ) @@ -150,3 +150,8 @@ log:info(logger.message_with_metadata("Finished", { module_ast = module_ast, other_nodes = other_nodes, })) + +filesystem.file_writer.write( + generator.teal_type_definitions.generate_teal(module_ast), + property.out_directory .. "/" .. "generated_from_ast" .. ".d.tl" +) From 5d456379543a5152982217d603bbd2294203ac33 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Thu, 4 May 2023 00:45:39 +0200 Subject: [PATCH 10/39] spec(module_doc): add basic test --- spec/example_spec.tl | 30 ------------------------------ spec/example_with_my_code_spec.tl | 10 ---------- spec/scraper/module_doc_spec.tl | 22 ++++++++++++++++++++++ 3 files changed, 22 insertions(+), 40 deletions(-) delete mode 100644 spec/example_spec.tl delete mode 100644 spec/example_with_my_code_spec.tl create mode 100644 spec/scraper/module_doc_spec.tl diff --git a/spec/example_spec.tl b/spec/example_spec.tl deleted file mode 100644 index 7448c7d..0000000 --- a/spec/example_spec.tl +++ /dev/null @@ -1,30 +0,0 @@ -local assert = require("luassert") - -describe("Busted unit testing framework", function() - describe("should be awesome", function() - it("should be easy to use", function() - assert.truthy "Yup." - end) - - it("should have lots of features", function() - -- deep check comparisons! - assert.same({ table = "great" }, { table = "great" }) - - -- or check by reference! - assert.not_equal({ table = "great" }, { table = "great" }) - - assert.truthy "this is a string" -- truthy: not false or nil - - assert.is_true(1 == 1) - - assert.falsy(nil) - assert.has_error(function() - error "Wat" - end, "Wat") - end) - - it("should provide some shortcuts to common functions", function() - assert.is_unique { { thing = 1 }, { thing = 2 }, { thing = 3 } } - end) - end) -end) diff --git a/spec/example_with_my_code_spec.tl b/spec/example_with_my_code_spec.tl deleted file mode 100644 index a510409..0000000 --- a/spec/example_with_my_code_spec.tl +++ /dev/null @@ -1,10 +0,0 @@ -local assert = require("luassert") -local utils = require("utils") - -describe("test", function() - it("has_item", function() - local t = {1, 2, 3} - assert.equal(utils.has_item(t, 1), 1) - assert.is_nil(utils.has_item(t, 4)) - end) -end) diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl new file mode 100644 index 0000000..c6d6571 --- /dev/null +++ b/spec/scraper/module_doc_spec.tl @@ -0,0 +1,22 @@ +local assert = require("luassert") +local scraper = require("scraper").module_doc + +local get_doc_from_page = scraper.get_doc_from_page + +describe("Scrap documentation", function() + it("should return a valid AST for an empty module", function() + local ast , nodes = get_doc_from_page("", "empty") + assert.same(ast, { + children = { + { + children = {}, + name = "Signal", + token = "enum", + } + }, + name = "Empty", + token = "module", + }) + assert.same(nodes, {}) + end) +end) From 65320173e5fe041adc714215d2ee5994ee0194ca Mon Sep 17 00:00:00 2001 From: Aire-One Date: Thu, 4 May 2023 00:46:27 +0200 Subject: [PATCH 11/39] spec(module_doc): add "Signals" test --- spec/scraper/module_doc_spec.tl | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl index c6d6571..b0432b4 100644 --- a/spec/scraper/module_doc_spec.tl +++ b/spec/scraper/module_doc_spec.tl @@ -19,4 +19,54 @@ describe("Scrap documentation", function() }) assert.same(nodes, {}) end) + + it("should produce Signal nodes", function() + local ast = get_doc_from_page([[ +

Signals

+
+
+ 馃敆 + widget::layout_changed + 路 Inherited from wibox.widget.base +
+
+
+ 馃敆 + widget::redraw_needed + 路 Inherited from wibox.widget.base +
+
+
+ ]], "signal") + assert.same(ast, { + children = { + { + children = { + { + name = "widget::layout_changed", + token = "identifier", + }, + { + name = "widget::redraw_needed", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + }, + name = "Signal", + token = "module", + }) + end) end) From 129f15ea9d1a536ea9f29beff75b7c3af5b82d78 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Thu, 4 May 2023 00:49:19 +0200 Subject: [PATCH 12/39] spec(module_doc): add "Object properties" tests --- .vscode/settings.json | 1 + spec/scraper/module_doc_spec.tl | 211 +++++++++++++++++++++++ src/awesomewm.d.tl/scraper/module_doc.tl | 2 +- 3 files changed, 213 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 008bada..87b906d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -37,6 +37,7 @@ "Stylua", "sublist", "tablex", + "tbody", "tmpl", "wibox", "woodpeckerci", diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl index b0432b4..b347cd4 100644 --- a/spec/scraper/module_doc_spec.tl +++ b/spec/scraper/module_doc_spec.tl @@ -20,6 +20,217 @@ describe("Scrap documentation", function() assert.same(nodes, {}) end) + it("should produce Variable and `property::` Signal nodes", function() + local ast = get_doc_from_page([[ +

+ Object properties +

+
+
+ 馃敆 + value + number + 路 1 signal +
+
+

Constraints:

+ + + + + + + + + + +
+ + Default value + + : 0
+ + Negative allowed + + : true
+ +
+
+ ]], "property_signal") + assert.same(ast, { + children = { + { + children = { + { + name = "property::value", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + { + name = "value", + types = { "number" }, + token = "variable", + } + }, + name = "Property_signal", + token = "module", + }) + end) + + it("should produce Enum nodes when an Object Property type is a String with constraints", function() + local ast = get_doc_from_page([[ +

+ Object properties +

+
+
+ 馃敆 + horizontal_fit_policy + string + 路 1 signal +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Default value + : "auto"
+ Valid values: +
+ + "auto" + + + : Honor the resize variable and preserve the aspect + ratio. +
+ + "none" + + : Do not resize at all.
+ + "fit" + + : Resize to the widget width.
+ +
+
+ ]], "property_enum") + assert.same(ast, { + children = { + { + children = { + { + name = "property::horizontal_fit_policy", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + { + children = { + { + name = "auto", + token = "identifier", + }, + { + name = "none", + token = "identifier", + }, + { + name = "fit", + token = "identifier", + }, + }, + name = "Horizontal_fit_policy", + token = "enum", + }, + { + name = "horizontal_fit_policy", + types = { "Horizontal_fit_policy" }, + token = "variable", + }, + }, + name = "Property_enum", + token = "module", + }) + end) + + it("should produce a `string` typed Variable node when a String Property has no constraint", function() + local ast = get_doc_from_page([[ +

+ Object properties +

+
+
+ 馃敆 + markup + string + 路 1 signal +
+
+ string +
+
+ ]], "property_string") + assert.same(ast, { + children = { + { + children = { + { + name = "property::markup", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + { + name = "markup", + types = { "string" }, + token = "variable", + } + }, + name = "Property_string", + token = "module", + }) + end) + it("should produce Signal nodes", function() local ast = get_doc_from_page([[

Signals

diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index 972f5d8..a58f2d8 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -82,7 +82,7 @@ end local function extract_property_constraints(property_constraint_node: scan.HTMLNode): { string } return scraper_utils.scrape( property_constraint_node:outer_html(), - "tr.see_also_sublist", + "tr.see_also_sublist i code", extract_node_text ) end From 9165e98db33b5169541bf43351ac17c979877b82 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Fri, 5 May 2023 00:04:25 +0200 Subject: [PATCH 13/39] =?UTF-8?q?chore:=20configure=20lua-local=20debugger?= =?UTF-8?q?=20for=20Teal=20=F0=9F=A7=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .busted | 9 ++++++++- .vscode/launch.json | 13 ++++++++++++- .vscode/settings.json | 13 +++++++------ justfile | 8 -------- src/awesomewm.d.tl/init.tl | 4 ++++ types/lldebugger.d.tl | 5 +++++ 6 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 types/lldebugger.d.tl diff --git a/.busted b/.busted index ff4c29f..1d410d2 100644 --- a/.busted +++ b/.busted @@ -1,3 +1,7 @@ +if os.getenv "LOCAL_LUA_DEBUGGER_VSCODE" == "1" then + require("lldebugger").start() +end + require("tl").loader() local version = _VERSION:match "%d+%.%d+" @@ -10,11 +14,14 @@ local function lua_module_paths(module_base_path) end return { - default = { + _all = { lpath = lua_module_paths("lua_modules/share/lua/" .. version) .. lua_module_paths "types" .. lua_module_paths "./src/awesomewm.d.tl", cpath = "lua_modules/lib/lua/" .. version .. "/?.so;", loaders = { "teal" }, }, + coverage = { + coverage = true, + }, } diff --git a/.vscode/launch.json b/.vscode/launch.json index cb54d96..c1f7800 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,18 @@ "program": { "command": "just" }, - "args": ["debug"] + "args": ["run"], + "scriptFiles": ["${workspaceFolder}/**/*.tl"] + }, + { + "name": "Debug specs", + "type": "lua-local", + "request": "launch", + "program": { + "command": "just" + }, + "args": ["test"], + "scriptFiles": ["${workspaceFolder}/**/*.tl"] } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 87b906d..8a4c47c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,10 +1,8 @@ { - "editor.formatOnSave": true, - "editor.formatOnPaste": true, "[markdown]": { - "editor.wordWrap": "on", + "editor.acceptSuggestionOnEnter": "off", "editor.renderWhitespace": "all", - "editor.acceptSuggestionOnEnter": "off" + "editor.wordWrap": "on" }, "cSpell.words": [ "aire-one", @@ -43,9 +41,12 @@ "woodpeckerci", "writefunction" ], + "debug.allowBreakpointsEverywhere": true, + "editor.formatOnPaste": true, + "editor.formatOnSave": true, "files.associations": { + ".busted": "lua", ".luacheckrc": "lua", - "*.rockspec": "lua", - ".busted": "lua" + "*.rockspec": "lua" } } diff --git a/justfile b/justfile index 370bcda..f2072b8 100644 --- a/justfile +++ b/justfile @@ -49,11 +49,3 @@ validate: test: busted - -# TODO : how to run a debugger on Teal code? -debug: - {{ lua }} debug.lua build/awesomewm.d.tl/init.lua -debug-gen: - cyan gen --output set_paths.lua --gen-compat "off" set_paths.tl - sed -i 's/"src\/awesomewm.d.tl"/"build\/awesomewm.d.tl"/g' set_paths.lua - stylua set_paths.lua diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index 605b341..3fcea57 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -1,3 +1,7 @@ +if os.getenv("LOCAL_LUA_DEBUGGER_VSCODE") == "1" then + require("lldebugger").start() +end + local crawler = require "crawler" local filesystem = require "filesystem" local generator = require "generator" diff --git a/types/lldebugger.d.tl b/types/lldebugger.d.tl new file mode 100644 index 0000000..5f2a85a --- /dev/null +++ b/types/lldebugger.d.tl @@ -0,0 +1,5 @@ +local record Lldebugger + start: function() +end + +return Lldebugger From 5e6ceb7e04a395a9524674d4ce2600216f4a6c13 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Fri, 5 May 2023 00:41:02 +0200 Subject: [PATCH 14/39] spec(module_doc): add "Static module functions" tests --- spec/scraper/module_doc_spec.tl | 268 ++++++++++++++++++++++++++++++++ 1 file changed, 268 insertions(+) diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl index b347cd4..4d07db1 100644 --- a/spec/scraper/module_doc_spec.tl +++ b/spec/scraper/module_doc_spec.tl @@ -280,4 +280,272 @@ describe("Scrap documentation", function() token = "module", }) end) + + it("should produce Function nodes", function() + local ast = get_doc_from_page([[ +

+ Static module functions +

+
+
+ 馃敆 + awesome.kill + (pid, sig) + -> boolean + +
+
+ Send a signal to a process. +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + +
NameType(s)Description
pid + integer + + Process identifier. 0 and negative values have special meaning. See + man 3 kill. +
sig + integer + + Signal number. See + awesome.unix_signal + for a list of signals. +
+

Returns:

+
    + boolean + true if the signal was successfully sent, else false +
+
+
+ ]], "awesome") -- The module name must be the same as the module name in the doc + assert.same(ast, { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + parameters = { + { + types = { "integer" }, + name = "pid", + token = "variable", + }, + { + types = { "integer" }, + name = "sig", + token = "variable", + }, + }, + return_types = { "boolean" }, + name = "kill", + token = "function", + } + }, + name = "Awesome", + token = "module", + }) + end) + + -- TODO : Fix the code then come back to this test, the current implementation is incomplete + -- it("should produce a Record node when the function parameter is a table", function() + -- local ast = get_doc_from_page([[ + --

+ -- Static module functions + --

+ --
+ --
+ -- 馃敆 + -- awful.screen.focused + -- {[args]} + -- -> nil or screen + -- + --
+ --
+ --

Parameters:

+ -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + -- + --
NameType(s)DescriptionDefault value
argsOptional + -- table + -- Undefined
clientOptional + -- boolean + -- + -- Use the client screen instead of the mouse screen. + -- + -- false + --
mouseOptional + -- boolean + -- Use the mouse screen + -- true + --
+ --

Returns:

+ --
    + -- optional + -- screen + -- The focused screen object, or + -- nil + -- in case no screen is present currently. + --
+ --
+ --
+ -- ]], "awful.screen") + -- print(require("inspect")(ast)) + -- assert.same(ast, { + -- children = { + -- { + -- children = {}, + -- name = "Signal", + -- token = "enum", + -- }, + -- { + -- parameters = { + -- { + -- children = { + -- { + -- types = { "boolean" }, + -- name = "client", + -- token = "variable", + -- }, + -- { + -- types = { "boolean" }, + -- name = "mouse", + -- token = "variable", + -- }, + -- }, + -- name = "Args", + -- token = "record", + -- }, + -- { + -- types = { "Focused_Args" }, + -- name = "args", + -- token = "variable", + -- }, + -- }, + -- return_types = { "screen" }, + -- name = "focused", + -- token = "function", + -- }, + -- }, + -- name = "Screen", + -- token = "module", + -- }) + -- end) + + it("should return Function nodes with the `other_nodes` list when the function module name doesn't match the module name", function() + local ast , other_nodes = get_doc_from_page([[ +

+ Static module functions +

+
+
+ 馃敆 + client.instances + () + -> integer + +
+
+ Get the number of instances. +

Returns:

+
    + integer + The number of client objects alive. +
+ +
+
+ ]], "awful.client") + assert.same(ast, { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + }, + name = "Client", + token = "module", + }) + assert.same(other_nodes, { + { + parameters = {}, + return_types = { "integer" }, + name = "client.instances", + token = "function", + } + }) + end) end) From efb54bf92b8b2135a4c7129bfb76ad680e165e92 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Mon, 8 May 2023 14:25:50 +0200 Subject: [PATCH 15/39] spec(module_doc): add "Object methods" tests --- spec/scraper/module_doc_spec.tl | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl index 4d07db1..f8f691f 100644 --- a/spec/scraper/module_doc_spec.tl +++ b/spec/scraper/module_doc_spec.tl @@ -231,6 +231,71 @@ describe("Scrap documentation", function() }) end) + it("should provide a Function node with the `self` as the first positional parameter", function() + local ast = get_doc_from_page([[ +

Object methods

+
+
+ 馃敆 + :swap (tag2) + + +
+
+

Parameters:

+ + + + + + + + + + + + + + + +
NameType(s)Description
tag2 + tag + The second tag
+
+
+ ]], "awful.tag") + assert.same(ast, { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + parameters = { + { + types = { "Tag" }, + name = "self", + token = "variable", + }, + { + types = { "tag" }, -- This needs to be fixed : tag -> Tag + name = "tag2", + token = "variable", + } + }, + return_types = {}, + name = "swap", + token = "function", + }, + }, + name = "Tag", + token = "module", + }) + end) + it("should produce Signal nodes", function() local ast = get_doc_from_page([[

Signals

From 3bf9ac4ebd8e2ac9bb77b50e73fa367d5d24309e Mon Sep 17 00:00:00 2001 From: Aire-One Date: Mon, 8 May 2023 23:12:16 +0200 Subject: [PATCH 16/39] chore: configure coverage --- .editorconfig | 8 +------- .gitignore | 1 + .luacov | 46 +++++++++++++++++++++++++++++++++++++++++++ .vscode/settings.json | 6 ++++++ justfile | 5 +++++ 5 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 .luacov diff --git a/.editorconfig b/.editorconfig index dc552b6..2ff1b1a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,13 +8,7 @@ charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true -[*.lua] -indent_size = 3 - -[*.{tl,d.tl}] -indent_size = 3 - -[.busted] +[*.{lua,tl,d.tl,busted,luacheckrc,luacov}] indent_size = 3 [justfile] diff --git a/.gitignore b/.gitignore index f6ca0ea..97ac9ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ lua_modules generated build +luacov diff --git a/.luacov b/.luacov new file mode 100644 index 0000000..381eb6d --- /dev/null +++ b/.luacov @@ -0,0 +1,46 @@ +-- -- Per https://github.com/lunarmodules/luacov/blob/c915f4b95e8df5dbfaf2b1f04f0b7da04506fc7a/src/luacov/reporter.lua#L102 +-- -- > The option includeuntestedfiles requires the lfs module (from luafilesystem) to be installed. +-- -- So I'm ok with including it here, since it's already an hard dependency. +-- local lfs = require "lfs" + +-- local function walk(dir, callback) +-- for file in lfs.dir(dir) do +-- if file ~= "." and file ~= ".." then +-- local path = dir .. "/" .. file +-- local attr = lfs.attributes(path) +-- if attr.mode == "directory" then +-- walk(path, callback) +-- else +-- callback(path) +-- end +-- end +-- end +-- end + +-- local teal_files = {} +-- for _, path in pairs { "spec", "src" } do +-- walk(path, function(file) +-- if file:match "%.tl$" then +-- table.insert(teal_files, file) +-- end +-- end) +-- end + +return { + statsfile = "luacov/luacov.stats.out", + reporter = "html", + reportfile = "luacov/luacov.report.html", + include = { + "spec", + "src", + }, + exclude = { + "lua_modules", + "spec/tlconfig.lua", + "src/awesome.d.tl/types", + "types", + }, + -- runreport = true, -- For some reason, it makes my computer crash + -- includeuntestedfiles = teal_files, + includeuntestedfiles = true, +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 8a4c47c..e0a991d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,6 +15,7 @@ "getcontent", "gitea", "htmlparser", + "includeuntestedfiles", "isdir", "justfile", "libc", @@ -22,6 +23,7 @@ "lpath", "Luacheck", "luacheckrc", + "luacov", "luafilesystem", "lualogging", "Luarocks", @@ -29,8 +31,11 @@ "luasocket", "mkdir", "modname", + "reportfile", "rockspec", + "runreport", "setopt", + "statsfile", "stringx", "Stylua", "sublist", @@ -47,6 +52,7 @@ "files.associations": { ".busted": "lua", ".luacheckrc": "lua", + ".luacov": "lua", "*.rockspec": "lua" } } diff --git a/justfile b/justfile index f2072b8..cb67890 100644 --- a/justfile +++ b/justfile @@ -49,3 +49,8 @@ validate: test: busted + +# Requires a patched version of luacov (https://github.com/lunarmodules/luacov/issues/98#issuecomment-1530491759) +coverage: + busted --coverage + luacov From ba51f09e475239060636978463412e2b6275cf19 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Mon, 8 May 2023 23:24:59 +0200 Subject: [PATCH 17/39] chore: rename Generator -> Printer --- src/awesomewm.d.tl/generator/init.tl | 6 ---- src/awesomewm.d.tl/init.tl | 12 +++---- .../{generator => printer}/global_env_def.tl | 2 +- src/awesomewm.d.tl/printer/init.tl | 6 ++++ .../module_init_definition.tl | 2 +- .../{generator => printer}/snippets.tl | 0 .../teal_type_definition.tl} | 36 +++++++++---------- 7 files changed, 32 insertions(+), 32 deletions(-) delete mode 100644 src/awesomewm.d.tl/generator/init.tl rename src/awesomewm.d.tl/{generator => printer}/global_env_def.tl (96%) create mode 100644 src/awesomewm.d.tl/printer/init.tl rename src/awesomewm.d.tl/{generator => printer}/module_init_definition.tl (93%) rename src/awesomewm.d.tl/{generator => printer}/snippets.tl (100%) rename src/awesomewm.d.tl/{generator/teal_type_definitions.tl => printer/teal_type_definition.tl} (72%) diff --git a/src/awesomewm.d.tl/generator/init.tl b/src/awesomewm.d.tl/generator/init.tl deleted file mode 100644 index 586a64b..0000000 --- a/src/awesomewm.d.tl/generator/init.tl +++ /dev/null @@ -1,6 +0,0 @@ -return { - global_env_def = require "generator.global_env_def", - module_init_definition = require "generator.module_init_definition", - snippets = require "generator.snippets", - teal_type_definitions = require "generator.teal_type_definitions", -} diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index 3fcea57..5fa2966 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -4,7 +4,7 @@ end local crawler = require "crawler" local filesystem = require "filesystem" -local generator = require "generator" +local printer = require "printer" local List = require "pl.List" local logger = require "logger" local Map = require "pl.Map" @@ -42,7 +42,7 @@ local function module_lists( end -- The module's children list produced can contain duplicates. --- We ignore them for now because they are dismissed when building a Map for the generator. +-- We ignore them for now because they are dismissed when building a Map for the printer. local function modules_tree(modules: List): Map> local tree: Map> = Map() for module in modules:iter() do @@ -79,7 +79,7 @@ end -- module_doc:fixup() -- module_doc:populate_requires() -- filesystem.file_writer.write( --- generator.teal_type_definitions.generate_teal(module_doc), +-- printer.teal_type_definitions.generate_teal(module_doc), -- output_path -- ) -- end @@ -128,7 +128,7 @@ end -- global_env_def:append(module_doc) -- end -- filesystem.file_writer.write( --- generator.global_env_def.generate_teal(global_env_def), +-- printer.global_env_def.generate_teal(global_env_def), -- property.out_directory .. "/global_env.d.tl" -- ) @@ -140,7 +140,7 @@ end -- requires:set(name, child) -- end -- filesystem.file_writer.write( --- generator.module_init_definition.generate_teal(requires), +-- printer.module_init_definition.generate_teal(requires), -- property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl" -- ) -- end @@ -156,6 +156,6 @@ log:info(logger.message_with_metadata("Finished", { })) filesystem.file_writer.write( - generator.teal_type_definitions.generate_teal(module_ast), + printer.teal_type_definition.printer(module_ast), property.out_directory .. "/" .. "generated_from_ast" .. ".d.tl" ) diff --git a/src/awesomewm.d.tl/generator/global_env_def.tl b/src/awesomewm.d.tl/printer/global_env_def.tl similarity index 96% rename from src/awesomewm.d.tl/generator/global_env_def.tl rename to src/awesomewm.d.tl/printer/global_env_def.tl index 47387d6..79b9250 100644 --- a/src/awesomewm.d.tl/generator/global_env_def.tl +++ b/src/awesomewm.d.tl/printer/global_env_def.tl @@ -2,7 +2,7 @@ local List = require "pl.List" local Module_Doc = require "entity.Module_Doc" local template = require "pl.template" local utils = require "utils" -local snippets = require "generator.snippets" +local snippets = require "printer.snippets" -- The long therm goal is to have so many `snippets.render_*` functions that -- we can render the whole file with the smallest template possible. diff --git a/src/awesomewm.d.tl/printer/init.tl b/src/awesomewm.d.tl/printer/init.tl new file mode 100644 index 0000000..1787676 --- /dev/null +++ b/src/awesomewm.d.tl/printer/init.tl @@ -0,0 +1,6 @@ +return { + global_env_def = require("printer.global_env_def"), + module_init_definition = require("printer.module_init_definition"), + snippets = require("printer.snippets"), + teal_type_definition = require("printer.teal_type_definition"), +} diff --git a/src/awesomewm.d.tl/generator/module_init_definition.tl b/src/awesomewm.d.tl/printer/module_init_definition.tl similarity index 93% rename from src/awesomewm.d.tl/generator/module_init_definition.tl rename to src/awesomewm.d.tl/printer/module_init_definition.tl index 20d8ed4..dca48f5 100644 --- a/src/awesomewm.d.tl/generator/module_init_definition.tl +++ b/src/awesomewm.d.tl/printer/module_init_definition.tl @@ -1,7 +1,7 @@ local Map = require "pl.Map" local template = require "pl.template" local utils = require "utils" -local snippets = require "generator.snippets" +local snippets = require "printer.snippets" -- The long therm goal is to have so many `snippets.render_*` functions that -- we can render the whole file with the smallest template possible. diff --git a/src/awesomewm.d.tl/generator/snippets.tl b/src/awesomewm.d.tl/printer/snippets.tl similarity index 100% rename from src/awesomewm.d.tl/generator/snippets.tl rename to src/awesomewm.d.tl/printer/snippets.tl diff --git a/src/awesomewm.d.tl/generator/teal_type_definitions.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl similarity index 72% rename from src/awesomewm.d.tl/generator/teal_type_definitions.tl rename to src/awesomewm.d.tl/printer/teal_type_definition.tl index 39393d1..82c7fd7 100644 --- a/src/awesomewm.d.tl/generator/teal_type_definitions.tl +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -20,23 +20,23 @@ local function render_types(types: { string }): string return ": " .. concatenate_strings(types) end -local record Node_Generator_Function +local record Node_Printer_Function on_node: function(node: Node): string before_node: nil | function(node: Node): string after_node: nil | function(node: Node): string end -- pre-declare functions to prevent forward reference errors -local generate_teal: function(node: Node): string -local generate_children: function(node: Node): string +local print_teal: function(node: Node): string +local print_children: function(node: Node): string -local node_generator : { Node.Token : Node_Generator_Function } = { +local node_printer : { Node.Token : Node_Printer_Function } = { ["module"] = { before_node = function(node: Node): string return "-- This file was auto-generated.\n\nlocal record " .. node.name .. "\n" end, on_node = function(node: Node): string - return generate_children(node) + return print_children(node) end, after_node = function(node: Node): string return "end\n\nreturn " .. node.name @@ -47,7 +47,7 @@ local node_generator : { Node.Token : Node_Generator_Function } = { return "local record " .. node.name end, on_node = function(node: Node): string - return generate_children(node) + return print_children(node) end, after_node = function(): string return "end" @@ -58,7 +58,7 @@ local node_generator : { Node.Token : Node_Generator_Function } = { return "enum " .. node.name .. "\n" end, on_node = function(node: Node): string - return generate_children(node) + return print_children(node) end, after_node = function(): string return "end" @@ -78,7 +78,7 @@ local node_generator : { Node.Token : Node_Generator_Function } = { on_node = function(node: Node): string local args = {} for _, parameter in ipairs(node.parameters) do - table.insert(args, generate_teal(parameter)) + table.insert(args, print_teal(parameter)) end return node.name .. ": function(" .. table.concat(args, ", ") .. ")" .. render_types(node.return_types) end, @@ -90,27 +90,27 @@ local node_generator : { Node.Token : Node_Generator_Function } = { } } -function generate_teal(node: Node): string - local generator = node_generator[node.token] +function print_teal(node: Node): string + local printer = node_printer[node.token] local generated = "" - if generator.before_node then - generated = (generator.before_node as function(node: Node): string)(node) + if printer.before_node then + generated = (printer.before_node as function(node: Node): string)(node) end - generated = generated .. generator.on_node(node) - if generator.after_node then - generated = generated .. (generator.after_node as function(node: Node): string)(node) + generated = generated .. printer.on_node(node) + if printer.after_node then + generated = generated .. (printer.after_node as function(node: Node): string)(node) end return generated end -function generate_children(node: Node): string +function print_children(node: Node): string local generated = "" for _, child in ast.iter_children(node) do - generated = generated .. generate_teal(child) .. "\n" + generated = generated .. print_teal(child) .. "\n" end return generated end return { - generate_teal = generate_teal, + printer = print_teal, } From 008985d5eaa17168d46b437723505d1d91b9163e Mon Sep 17 00:00:00 2001 From: Aire-One Date: Mon, 8 May 2023 23:48:08 +0200 Subject: [PATCH 18/39] chore(spec): defined expected with type safety --- spec/scraper/module_doc_spec.tl | 50 ++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl index f8f691f..2ed19fc 100644 --- a/spec/scraper/module_doc_spec.tl +++ b/spec/scraper/module_doc_spec.tl @@ -1,12 +1,13 @@ local assert = require("luassert") -local scraper = require("scraper").module_doc +local type Node = require("types.Node") +local scraper = require("scraper.module_doc") local get_doc_from_page = scraper.get_doc_from_page describe("Scrap documentation", function() it("should return a valid AST for an empty module", function() local ast , nodes = get_doc_from_page("", "empty") - assert.same(ast, { + local expected : Node = { children = { { children = {}, @@ -16,8 +17,9 @@ describe("Scrap documentation", function() }, name = "Empty", token = "module", - }) - assert.same(nodes, {}) + } + assert.same(expected, ast) + assert.same({}, nodes) end) it("should produce Variable and `property::` Signal nodes", function() @@ -57,7 +59,7 @@ describe("Scrap documentation", function() ]], "property_signal") - assert.same(ast, { + local expected : Node = { children = { { children = { @@ -77,7 +79,8 @@ describe("Scrap documentation", function() }, name = "Property_signal", token = "module", - }) + } + assert.same(expected, ast) end) it("should produce Enum nodes when an Object Property type is a String with constraints", function() @@ -150,7 +153,7 @@ describe("Scrap documentation", function() ]], "property_enum") - assert.same(ast, { + local expected : Node = { children = { { children = { @@ -188,7 +191,8 @@ describe("Scrap documentation", function() }, name = "Property_enum", token = "module", - }) + } + assert.same(expected, ast) end) it("should produce a `string` typed Variable node when a String Property has no constraint", function() @@ -208,7 +212,7 @@ describe("Scrap documentation", function() ]], "property_string") - assert.same(ast, { + local expected : Node = { children = { { children = { @@ -228,7 +232,8 @@ describe("Scrap documentation", function() }, name = "Property_string", token = "module", - }) + } + assert.same(expected, ast) end) it("should provide a Function node with the `self` as the first positional parameter", function() @@ -266,7 +271,7 @@ describe("Scrap documentation", function() ]], "awful.tag") - assert.same(ast, { + local expected : Node = { children = { { children = {}, @@ -293,7 +298,8 @@ describe("Scrap documentation", function() }, name = "Tag", token = "module", - }) + } + assert.same(expected, ast) end) it("should produce Signal nodes", function() @@ -324,7 +330,7 @@ describe("Scrap documentation", function()
]], "signal") - assert.same(ast, { + local expected : Node = { children = { { children = { @@ -343,7 +349,8 @@ describe("Scrap documentation", function() }, name = "Signal", token = "module", - }) + } + assert.same(expected, ast) end) it("should produce Function nodes", function() @@ -409,7 +416,7 @@ describe("Scrap documentation", function() ]], "awesome") -- The module name must be the same as the module name in the doc - assert.same(ast, { + local expected : Node = { children = { { children = {}, @@ -436,7 +443,8 @@ describe("Scrap documentation", function() }, name = "Awesome", token = "module", - }) + } + assert.same(expected, ast) end) -- TODO : Fix the code then come back to this test, the current implementation is incomplete @@ -593,7 +601,7 @@ describe("Scrap documentation", function() ]], "awful.client") - assert.same(ast, { + local expected_ast : Node = { children = { { children = {}, @@ -603,14 +611,16 @@ describe("Scrap documentation", function() }, name = "Client", token = "module", - }) - assert.same(other_nodes, { + } + assert.same(expected_ast, ast) + local expected_other_nodes : { Node } = { { parameters = {}, return_types = { "integer" }, name = "client.instances", token = "function", } - }) + } + assert.same(expected_other_nodes, other_nodes) end) end) From 1ccb6d0ce948ad0ba0f2aa113350724f2ef0fc58 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Tue, 9 May 2023 00:27:29 +0200 Subject: [PATCH 19/39] spec(teal_type_definition): add basic test --- spec/printer/teal_type_definition_spec.tl | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 spec/printer/teal_type_definition_spec.tl diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl new file mode 100644 index 0000000..7e49a7c --- /dev/null +++ b/spec/printer/teal_type_definition_spec.tl @@ -0,0 +1,35 @@ +local assert = require("luassert") +local type Node = require("types.Node") +local stringx = require("pl.stringx") +local teal_type_definition_printer = require("printer.teal_type_definition") + +local printer = teal_type_definition_printer.printer + +-- We need to remove the last newline inserted by Penlight's dedent +local function dedent(str: string): string + return (stringx.dedent(str):sub(1, -3)) +end + +local function gen(ast: Node, expected_code: string): function() + return function() + local generated = printer(ast) + assert.same(dedent(expected_code), generated) + end +end + +describe("Print Teal type definition", function() + it("should print a simple module type definition", gen( + { + children = {}, + name = "Empty", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Empty + end + + return Empty + ]])) +end) From eca034828c8dd0a3a319567f58337ec7bb68a309 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Tue, 9 May 2023 00:29:06 +0200 Subject: [PATCH 20/39] chore(just): add pattern parameter to test command --- justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index cb67890..d35a481 100644 --- a/justfile +++ b/justfile @@ -47,8 +47,8 @@ validate: awesomerc.tl # `find . -type f -iname '*.d.tl' | xargs` -test: - busted +test PATTERN="_spec": + busted --pattern={{ PATTERN }} # Requires a patched version of luacov (https://github.com/lunarmodules/luacov/issues/98#issuecomment-1530491759) coverage: From 6410cf6a75b43ea60d93ff6794be3229d27b663f Mon Sep 17 00:00:00 2001 From: Aire-One Date: Tue, 9 May 2023 00:40:53 +0200 Subject: [PATCH 21/39] spec(teal_type_definition): test Enum node --- spec/printer/teal_type_definition_spec.tl | 36 ++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl index 7e49a7c..a54c849 100644 --- a/spec/printer/teal_type_definition_spec.tl +++ b/spec/printer/teal_type_definition_spec.tl @@ -7,7 +7,7 @@ local printer = teal_type_definition_printer.printer -- We need to remove the last newline inserted by Penlight's dedent local function dedent(str: string): string - return (stringx.dedent(str):sub(1, -3)) + return (stringx.dedent(str):sub(1, -3):gsub("\n *", "\n")) -- we remove indentation for now, the printer should be fixed soon end local function gen(ast: Node, expected_code: string): function() @@ -32,4 +32,38 @@ describe("Print Teal type definition", function() return Empty ]])) + + it("should print an Enum exposed by the module", gen( + { + children = { + { + children = { + { + name = "widget::layout_changed", + token = "identifier", + }, + { + name = "widget::redraw_needed", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + }, + name = "Signal_Module", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Signal_Module + enum Signal + "widget::layout_changed" + "widget::redraw_needed" + end + end + + return Signal_Module + ]])) end) From a1b6a383bb3e422317a8d607802cb4e60b5953f9 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 13 May 2023 00:19:49 +0200 Subject: [PATCH 22/39] spec(teal_type_definition): test Variable node --- spec/printer/teal_type_definition_spec.tl | 24 ++++++++++++++++++- .../printer/teal_type_definition.tl | 17 ++++--------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl index a54c849..d22dc32 100644 --- a/spec/printer/teal_type_definition_spec.tl +++ b/spec/printer/teal_type_definition_spec.tl @@ -17,7 +17,7 @@ local function gen(ast: Node, expected_code: string): function() end end -describe("Print Teal type definition", function() +describe("Teal type definition Printer", function() it("should print a simple module type definition", gen( { children = {}, @@ -66,4 +66,26 @@ describe("Print Teal type definition", function() return Signal_Module ]])) + + it("should print a property exposed by the module", gen( + { + children = { + { + name = "text", + types = { "string", "nil" }, + token = "variable", + } + }, + name = "Property_Module", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Property_Module + text: string | nil + end + + return Property_Module + ]])) end) diff --git a/src/awesomewm.d.tl/printer/teal_type_definition.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl index 82c7fd7..7958d39 100644 --- a/src/awesomewm.d.tl/printer/teal_type_definition.tl +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -1,23 +1,16 @@ -local ast = require "ast" -local logger = require "logger" -local type Node = require "types.Node" +local ast = require("ast") +local logger = require("logger") +local type Node = require("types.Node") +local stringx = require("pl.stringx") local log = logger.log("scraper") -local function concatenate_strings(strings: { string }): string - local result = "" - for _, str in ipairs(strings) do - result = result .. str - end - return result -end - local function render_types(types: { string }): string if not types or #types == 0 then return "" end - return ": " .. concatenate_strings(types) + return ": " .. stringx.join(" | ", types) end local record Node_Printer_Function From 8737c9dd28fe3e391663aa058286f79dbd0ae18d Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 13 May 2023 00:24:58 +0200 Subject: [PATCH 23/39] spec(teal_type_definition): test Function node --- spec/printer/teal_type_definition_spec.tl | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl index d22dc32..6a717b8 100644 --- a/spec/printer/teal_type_definition_spec.tl +++ b/spec/printer/teal_type_definition_spec.tl @@ -88,4 +88,38 @@ describe("Teal type definition Printer", function() return Property_Module ]])) + + it("should print a function exposed by the module", gen( + { + children = { + { + parameters = { + { + types = { "integer" }, + name = "pid", + token = "variable", + }, + { + types = { "integer" }, + name = "sig", + token = "variable", + }, + }, + return_types = { "boolean" }, + name = "kill", + token = "function", + }, + }, + name = "Function_Module", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Function_Module + kill: function(pid: integer, sig: integer): boolean + end + + return Function_Module + ]])) end) From b58384e65b9be0a8ac2cd500654492222ce3cfea Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 13 May 2023 00:29:47 +0200 Subject: [PATCH 24/39] spec(teal_type_definition): test Record node --- spec/printer/teal_type_definition_spec.tl | 23 +++++++++++++++++++ .../printer/teal_type_definition.tl | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl index 6a717b8..9c37e19 100644 --- a/spec/printer/teal_type_definition_spec.tl +++ b/spec/printer/teal_type_definition_spec.tl @@ -122,4 +122,27 @@ describe("Teal type definition Printer", function() return Function_Module ]])) + + it("should print nested Record exposed by the module", gen( + { + children = { + { + children = {}, + name = "Nested", + token = "record", + } + }, + name = "Nested_Module", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Nested_Module + record Nested + end + end + + return Nested_Module + ]])) end) diff --git a/src/awesomewm.d.tl/printer/teal_type_definition.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl index 7958d39..f43981b 100644 --- a/src/awesomewm.d.tl/printer/teal_type_definition.tl +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -37,7 +37,7 @@ local node_printer : { Node.Token : Node_Printer_Function } = { }, ["record"] = { before_node = function(node: Node): string - return "local record " .. node.name + return "record " .. node.name .. "\n" end, on_node = function(node: Node): string return print_children(node) From c2e0da3ba6967de2d4780a79eb1bc0cc91c235ab Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 13 May 2023 03:42:27 +0200 Subject: [PATCH 25/39] fix(module_doc): named parameter table detection --- spec/scraper/module_doc_spec.tl | 429 ++++++++++++++++------- src/awesomewm.d.tl/scraper/module_doc.tl | 104 +++--- src/awesomewm.d.tl/scraper/utils.tl | 11 +- src/awesomewm.d.tl/utils.tl | 6 + 4 files changed, 379 insertions(+), 171 deletions(-) diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl index 2ed19fc..b56b53d 100644 --- a/spec/scraper/module_doc_spec.tl +++ b/spec/scraper/module_doc_spec.tl @@ -447,131 +447,310 @@ describe("Scrap documentation", function() assert.same(expected, ast) end) - -- TODO : Fix the code then come back to this test, the current implementation is incomplete - -- it("should produce a Record node when the function parameter is a table", function() - -- local ast = get_doc_from_page([[ - --

- -- Static module functions - --

- --
- --
- -- 馃敆 - -- awful.screen.focused - -- {[args]} - -- -> nil or screen - -- - --
- --
- --

Parameters:

- -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - -- - --
NameType(s)DescriptionDefault value
argsOptional - -- table - -- Undefined
clientOptional - -- boolean - -- - -- Use the client screen instead of the mouse screen. - -- - -- false - --
mouseOptional - -- boolean - -- Use the mouse screen - -- true - --
- --

Returns:

- --
    - -- optional - -- screen - -- The focused screen object, or - -- nil - -- in case no screen is present currently. - --
- --
- --
- -- ]], "awful.screen") - -- print(require("inspect")(ast)) - -- assert.same(ast, { - -- children = { - -- { - -- children = {}, - -- name = "Signal", - -- token = "enum", - -- }, - -- { - -- parameters = { - -- { - -- children = { - -- { - -- types = { "boolean" }, - -- name = "client", - -- token = "variable", - -- }, - -- { - -- types = { "boolean" }, - -- name = "mouse", - -- token = "variable", - -- }, - -- }, - -- name = "Args", - -- token = "record", - -- }, - -- { - -- types = { "Focused_Args" }, - -- name = "args", - -- token = "variable", - -- }, - -- }, - -- return_types = { "screen" }, - -- name = "focused", - -- token = "function", - -- }, - -- }, - -- name = "Screen", - -- token = "module", - -- }) - -- end) + it("should produce a Record node when a function parameter is a named-parameter-table", function() + local ast = get_doc_from_page([[ +

+ Static module functions +

+
+
+ 馃敆 + awful.screen.focused + {[args]} + -> nil or screen + +
+
+

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameType(s)DescriptionDefault value
argsOptional + table + Undefined
clientOptional + boolean + + Use the client screen instead of the mouse screen. + + false +
mouseOptional + boolean + Use the mouse screen + true +
+

Returns:

+
    + optional + screen + The focused screen object, or + nil + in case no screen is present currently. +
+
+
+ ]], "awful.screen") + assert.same(ast, { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + children = { + { + types = { "boolean" }, + name = "client", + token = "variable", + }, + { + types = { "boolean" }, + name = "mouse", + token = "variable", + } + }, + name = "Focused_Args", + token = "record", + }, + { + parameters = { + { + types = { "Focused_Args" }, + name = "args", + token = "variable", + }, + }, + return_types = { "screen" }, + name = "focused", + token = "function", + }, + }, + name = "Screen", + token = "module", + }) + end) + + it("should go back to a table typed parameter when the record is empty", function() + local ast = get_doc_from_page([[ +

Static module functions

+
+
+ 馃敆 + gears.table.crush (target, source, raw) + -> table + + +
+
+

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameType(s)DescriptionDefault value
targettable The target table. Values from source will be copied + into this table.Not applicable
sourcetable The source table. Its values will be copied into + target.Not applicable
rawOptionalbool If true, values will be assigned with rawset. + This will bypass metamethods on target.false
+

Returns:

+
    + table + The target table. +
+
+
+ ]], "gears.table") + local expected : Node = { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + parameters = { + { + types = { "table" }, + name = "target", + token = "variable", + }, + { + types = { "table" }, + name = "source", + token = "variable", + }, + { + types = { "bool" }, + name = "raw", + token = "variable", + } + }, + return_types = { "table" }, + name = "crush", + token = "function", + } + }, + name = "Table", + token = "module", + } + assert.same(expected, ast) + end) + + it("should go back to a table typed parameter when the record is empty and it's the last parameter", function() + local ast = get_doc_from_page([[ +

Object methods

+
+
+ 馃敆 + :tags + (tags_table) + -> table + 路 1 signal +
+
+

Parameters:

+ + + + + + + + + + + + + + + +
NameType(s)Description
tags_table + table + + A table with tags to set, or nil to get the current + tags. +
+

Returns:

+
    + table + A table with all tags. +
+
+
+ ]], "awful.client") + local expected : Node = { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + parameters = { + { + types = { "Client" }, + name = "self", + token = "variable", + }, + { + types = { "table" }, + name = "tags_table", + token = "variable", + }, + }, + return_types = { "table" }, + name = "tags", + token = "function", + }, + }, + name = "Client", + token = "module", + } + assert.same(expected, ast) + end) it("should return Function nodes with the `other_nodes` list when the function module name doesn't match the module name", function() local ast , other_nodes = get_doc_from_page([[ diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index a58f2d8..9ef4c64 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -34,45 +34,63 @@ local function extract_item_name(item_name_node: scan.HTMLNode): string, string return utils.sanitize_string(name), module_name and utils.sanitize_string(module_name) or nil end -local function extract_function_parameters(table_html: string): { Node } - local current_record_parameter: Node = nil +local function extract_function_parameters(table_html: string, function_name: string): { Node }, { Node } + local parameters_types : { Node } = {} + local is_populating_parameters_types: boolean = false + local current_field: Node = nil - return scraper_utils.scrape(table_html, "tr", function(tr: scan.HTMLNode): Node - local tr_html = tr:outer_html() - local name_node = scraper_utils.find(tr_html, "span.parameter")[1] - local types_node = scraper_utils.find(tr_html, "span.types")[1] - if not name_node or not types_node then - return nil - end + local parameters = scraper_utils.scrape(table_html, "tr", function(tr: scan.HTMLNode): Node + local tr_html = tr:outer_html() + local name_node = scraper_utils.find(tr_html, "span.parameter")[1] + local types_node = scraper_utils.find(tr_html, "span.types")[1] + if not name_node or not types_node then + return nil + end - local name = extract_node_text(name_node) - local types = parse_parameter_types(extract_node_text(types_node)) - - if tr.attr ~= nil and tr.attr.class == "see_also_sublist" and current_record_parameter then - local field = ast.create_node("variable", name) - field.types = types - table.insert(current_record_parameter.children, field) - return nil - end - - -- We wrongly tried to convert a table to a record - if current_record_parameter then - current_record_parameter.token = "variable" - current_record_parameter.name = utils.lowercase(current_record_parameter.name) - current_record_parameter.types = { "table" } - current_record_parameter.children = nil - current_record_parameter = nil - end - - if #types == 1 and types[1] == "table" then - current_record_parameter = ast.create_node("record", utils.capitalize(name)) - return current_record_parameter - end + local name = extract_node_text(name_node) + local types = parse_parameter_types(extract_node_text(types_node)) + -- Add a field to the current parameter type record + if tr.attr ~= nil and tr.attr.class == "see_also_sublist" and is_populating_parameters_types then local field = ast.create_node("variable", name) field.types = types - return field - end) + table.insert(parameters_types[#parameters_types].children, field) + return nil + end + + -- Still here and we are populating the parameter type record ? + -- Then oops, we wrongly tried to convert a table to a record + if is_populating_parameters_types then + table.remove(parameters_types, #parameters_types) + is_populating_parameters_types = false + current_field.types = { "table" } + end + + -- Otherwise, add a new parameter + local field = ast.create_node("variable", name) + field.types = types + current_field = field + + -- If the parameter is a table, then we try to convert it to a record + if #types == 1 and types[1] == "table" then + local record_name = string.format( + "%s_%s", + utils.capitalize(function_name), + utils.capitalize(name)) + table.insert(parameters_types, ast.create_node("record", record_name)) + is_populating_parameters_types = true + field.types = { record_name } + end + + return field + end) + + if is_populating_parameters_types and #parameters_types[#parameters_types].children == 0 then + table.remove(parameters_types, #parameters_types) + current_field.types = { "table" } + end + + return parameters, parameters_types end local function extract_function_return_types(ol_html: string): { string } @@ -109,9 +127,11 @@ local function extract_section_functions(dl: string, module_name: string | nil): local body_html = nodes[list_query_selectors.body]:outer_html() local parameter_node = scraper_utils.find(body_html, "table")[1] - function_node.parameters = parameter_node and - extract_function_parameters(parameter_node:outer_html()) or - {} + local parameters, parameters_types: { Node }, { Node } = {}, {} + if parameter_node then + parameters, parameters_types = extract_function_parameters(parameter_node:outer_html(), function_name) + end + function_node.parameters = parameters local return_node = scraper_utils.find(body_html, "ol")[1] function_node.return_types = return_node and @@ -120,8 +140,10 @@ local function extract_section_functions(dl: string, module_name: string | nil): if module_name and function_module_name and module_name ~= function_module_name then function_node.name = function_module_name .. "." .. function_node.name + utils.spread(other_functions, parameters_types) table.insert(other_functions, function_node) else + utils.spread(functions, parameters_types) table.insert(functions, function_node) end end @@ -203,9 +225,11 @@ local section_scrapers : { Section : function(html: string, record_name: ["Object methods"] = function(html: string, record_name: string): { Node }, { Node }, { string } local methods = extract_section_functions(html) for _, method in ipairs(methods) do - local self_parameter = ast.create_node("variable", "self") - self_parameter.types = { record_name } - table.insert(method.parameters, 1, self_parameter) + if method.token == "function" then + local self_parameter = ast.create_node("variable", "self") + self_parameter.types = { record_name } + table.insert(method.parameters, 1, self_parameter) + end end return methods, {}, {} end, diff --git a/src/awesomewm.d.tl/scraper/utils.tl b/src/awesomewm.d.tl/scraper/utils.tl index 8454e03..271ed0d 100644 --- a/src/awesomewm.d.tl/scraper/utils.tl +++ b/src/awesomewm.d.tl/scraper/utils.tl @@ -15,15 +15,14 @@ function scraper_utils.scrape(html: string, query_selector: string, extract_c scanner.scan_html(html, function(stack: scan.NodeStack) if stack:is(query_selector) then local node = stack:current() - local success, info_or_error = pcall(extract_callback, node) + local success, ret_or_error = pcall(extract_callback, node) if not success then - local error_message = info_or_error as string - log:error(logger.message_with_metadata("Extraction error", { error = error_message })) - else - local info = info_or_error as T - table.insert(ret, info) + log:error(logger.message_with_metadata("Extraction error", { error = ret_or_error as string })) + return end + + table.insert(ret, ret_or_error as T) end end) diff --git a/src/awesomewm.d.tl/utils.tl b/src/awesomewm.d.tl/utils.tl index 824b9dc..6c695bc 100644 --- a/src/awesomewm.d.tl/utils.tl +++ b/src/awesomewm.d.tl/utils.tl @@ -76,4 +76,10 @@ function utils.do_or_fail(func: function(...: any): (T | nil, string), ... return res as T -- promote to T since pcall succeeded at this point end +function utils.spread(t: { T }, i: { T }) + for _, v in ipairs(i) do + table.insert(t, v) + end +end + return utils From 8928e67f24435d8430b4a4a74b3f129c1219189e Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 13 May 2023 14:33:10 +0200 Subject: [PATCH 26/39] feat(printer): indent generated code --- .vscode/settings.json | 2 + spec/printer/teal_type_definition_spec.tl | 2 +- .../printer/teal_type_definition.tl | 126 +++++++++++++----- 3 files changed, 92 insertions(+), 38 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index e0a991d..f071b81 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,6 +11,7 @@ "ansicolors", "awesomewm", "buildx", + "concat", "dryrun", "getcontent", "gitea", @@ -33,6 +34,7 @@ "modname", "reportfile", "rockspec", + "rstrip", "runreport", "setopt", "statsfile", diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl index 9c37e19..1174c1e 100644 --- a/spec/printer/teal_type_definition_spec.tl +++ b/spec/printer/teal_type_definition_spec.tl @@ -7,7 +7,7 @@ local printer = teal_type_definition_printer.printer -- We need to remove the last newline inserted by Penlight's dedent local function dedent(str: string): string - return (stringx.dedent(str):sub(1, -3):gsub("\n *", "\n")) -- we remove indentation for now, the printer should be fixed soon + return (stringx.dedent(str):sub(1, -2)) end local function gen(ast: Node, expected_code: string): function() diff --git a/src/awesomewm.d.tl/printer/teal_type_definition.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl index f43981b..4f089e4 100644 --- a/src/awesomewm.d.tl/printer/teal_type_definition.tl +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -9,97 +9,149 @@ local function render_types(types: { string }): string if not types or #types == 0 then return "" end + return ": " .. table.concat(types, " | ") +end - return ": " .. stringx.join(" | ", types) +local function dedent(str: string): string + return stringx.dedent(str):sub(1, -2) +end + +local function render_code(code: string, indent_level: integer): string + if not code or code == "" then + return "" + end + + local generated = "" + for line in stringx.lines(dedent(code)) do + generated = generated .. stringx.rstrip(string.rep(" ", 3 * indent_level) .. line) .. "\n" + end + return generated end local record Node_Printer_Function - on_node: function(node: Node): string - before_node: nil | function(node: Node): string - after_node: nil | function(node: Node): string + on_node: function(node: Node, indent_level: integer): string, integer + before_node: nil | function(node: Node, indent_level: integer): string, integer + after_node: nil | function(node: Node, indent_level: integer): string, integer end -- pre-declare functions to prevent forward reference errors -local print_teal: function(node: Node): string +local print_teal: function(node: Node, indent_level: integer | nil): string local print_children: function(node: Node): string local node_printer : { Node.Token : Node_Printer_Function } = { ["module"] = { - before_node = function(node: Node): string - return "-- This file was auto-generated.\n\nlocal record " .. node.name .. "\n" + before_node = function(node: Node, indent_level: integer): string, integer + return render_code( + string.format( + [[ + -- This file was auto-generated. + + local record %s + ]], + node.name), + indent_level), indent_level + 1 end, - on_node = function(node: Node): string - return print_children(node) + on_node = function(node: Node, indent_level: integer): string, integer + return render_code(print_children(node), indent_level), indent_level end, - after_node = function(node: Node): string - return "end\n\nreturn " .. node.name + after_node = function(node: Node, indent_level: integer): string, integer + return render_code("end", indent_level - 1) .. + "\n" .. + render_code( + string.format("return %s", node.name), + indent_level - 1 + ), indent_level - 1 end, }, ["record"] = { - before_node = function(node: Node): string - return "record " .. node.name .. "\n" + before_node = function(node: Node, indent_level: integer): string, integer + return render_code( + string.format( + "record %s", + node.name), + indent_level), indent_level + 1 end, - on_node = function(node: Node): string - return print_children(node) + on_node = function(node: Node, indent_level: integer): string, integer + return render_code(print_children(node), indent_level), indent_level end, - after_node = function(): string - return "end" + after_node = function(_: Node, indent_level: integer): string, integer + return render_code("end", indent_level - 1), indent_level - 1 end, }, ["enum"] = { - before_node = function(node: Node): string - return "enum " .. node.name .. "\n" + before_node = function(node: Node, indent_level: integer): string, integer + return render_code( + string.format( + "enum %s", + node.name), + indent_level), indent_level + 1 end, - on_node = function(node: Node): string - return print_children(node) + on_node = function(node: Node, indent_level: integer): string, integer + return render_code(print_children(node), indent_level), indent_level end, - after_node = function(): string - return "end" + after_node = function(_: Node, indent_level: integer): string, integer + return render_code("end", indent_level - 1), indent_level - 1 end, }, ["identifier"] = { - on_node = function(node: Node): string - return "\"" .. node.name .. "\"" + on_node = function(node: Node, indent_level: integer): string, integer + return render_code(string.format('"%s"', node.name), indent_level), indent_level end, }, ["variable"] = { - on_node = function(node: Node): string - return node.name .. render_types(node.types) + on_node = function(node: Node, indent_level: integer): string, integer + return render_code( + string.format( + "%s%s", + node.name, + render_types(node.types)), + indent_level), indent_level end, }, ["function"] = { - on_node = function(node: Node): string + on_node = function(node: Node, indent_level: integer): string, integer local args = {} for _, parameter in ipairs(node.parameters) do - table.insert(args, print_teal(parameter)) + table.insert(args, print_teal(parameter):sub(1, -2)) -- need to remove the newline ending end - return node.name .. ": function(" .. table.concat(args, ", ") .. ")" .. render_types(node.return_types) + return render_code( + string.format( + "%s: function(%s)%s", + node.name, + table.concat(args, ", "), + render_types(node.return_types)), + indent_level), indent_level end, }, ["metamethod"] = { - on_node = function(): string + on_node = function(): string, integer log:warn("Metamethods are not supported yet") end, } } -function print_teal(node: Node): string +function print_teal(node: Node, indent_level: integer | nil): string + indent_level = indent_level or 0 local printer = node_printer[node.token] local generated = "" + local full_generated = "" if printer.before_node then - generated = (printer.before_node as function(node: Node): string)(node) + generated, indent_level = (printer.before_node as function(Node, integer): string, integer)(node, indent_level) + full_generated = generated end - generated = generated .. printer.on_node(node) + generated, indent_level = printer.on_node(node, indent_level) + full_generated = full_generated .. generated if printer.after_node then - generated = generated .. (printer.after_node as function(node: Node): string)(node) + generated, indent_level = (printer.after_node as function(Node, integer): string, integer)(node, indent_level) + full_generated = full_generated .. generated end - return generated + return full_generated end function print_children(node: Node): string local generated = "" for _, child in ast.iter_children(node) do - generated = generated .. print_teal(child) .. "\n" + generated = generated .. print_teal(child) end return generated end From 02d82f6518a2ef1fe1d4a514645c770ebb26f1a8 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 14 May 2023 13:04:45 +0200 Subject: [PATCH 27/39] chore: update the `Debug spec` command --- .vscode/launch.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index c1f7800..dad13e8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -18,7 +18,10 @@ "program": { "command": "just" }, - "args": ["test"], + "args": [ + "test", + "_spec" // Adapt to the spec to debug + ], "scriptFiles": ["${workspaceFolder}/**/*.tl"] } ] From 37742f795d0c6161b4928ba2dac9f8750d2a10a7 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sat, 20 May 2023 15:14:02 +0200 Subject: [PATCH 28/39] feat(Visitors): implement type_mapping --- src/awesomewm.d.tl/ast.tl | 11 +++ src/awesomewm.d.tl/init.tl | 7 ++ src/awesomewm.d.tl/types/{Dag.d.tl => Dag.tl} | 0 .../types/{Node.d.tl => Node.tl} | 0 src/awesomewm.d.tl/types/Visitor.tl | 7 ++ src/awesomewm.d.tl/visitors/type_mapping.tl | 68 +++++++++++++++++++ 6 files changed, 93 insertions(+) rename src/awesomewm.d.tl/types/{Dag.d.tl => Dag.tl} (100%) rename src/awesomewm.d.tl/types/{Node.d.tl => Node.tl} (100%) create mode 100644 src/awesomewm.d.tl/types/Visitor.tl create mode 100644 src/awesomewm.d.tl/visitors/type_mapping.tl diff --git a/src/awesomewm.d.tl/ast.tl b/src/awesomewm.d.tl/ast.tl index 0f8a608..b74390e 100644 --- a/src/awesomewm.d.tl/ast.tl +++ b/src/awesomewm.d.tl/ast.tl @@ -59,10 +59,21 @@ local function create_node(token: Node.Token, name: string): Node end local function iter_children(node: Node): function(): integer, Node + if node.children == nil then + return function(): integer, Node end + end return ipairs(node.children) end +local function in_order_visitor(node: Node, visitor: function(Node)) + for _, child in iter_children(node) do + in_order_visitor(child, visitor) + end + visitor(node) +end + return { create_node = create_node, iter_children = iter_children, + in_order_visitor = in_order_visitor, } diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index 5fa2966..f1bc8c2 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -2,6 +2,7 @@ if os.getenv("LOCAL_LUA_DEBUGGER_VSCODE") == "1" then require("lldebugger").start() end +local ast = require "ast" local crawler = require "crawler" local filesystem = require "filesystem" local printer = require "printer" @@ -10,9 +11,11 @@ local logger = require "logger" local Map = require "pl.Map" local Module_Doc = require "entity.Module_Doc" local Module_Info = require "entity.Module_Info" +local type Node = require "types.Node" local property = require "property" local scraper = require "scraper" local stringx = require "pl.stringx" +local type_mapping = require "visitors.type_mapping" local utils = require "utils" local log = logger.log("main") @@ -150,6 +153,10 @@ local module_ast, other_nodes = scraper.module_doc.get_doc_from_page( "awful.tag" ) +ast.in_order_visitor(module_ast, function(node: Node) + type_mapping.visit(node) +end) + log:info(logger.message_with_metadata("Finished", { module_ast = module_ast, other_nodes = other_nodes, diff --git a/src/awesomewm.d.tl/types/Dag.d.tl b/src/awesomewm.d.tl/types/Dag.tl similarity index 100% rename from src/awesomewm.d.tl/types/Dag.d.tl rename to src/awesomewm.d.tl/types/Dag.tl diff --git a/src/awesomewm.d.tl/types/Node.d.tl b/src/awesomewm.d.tl/types/Node.tl similarity index 100% rename from src/awesomewm.d.tl/types/Node.d.tl rename to src/awesomewm.d.tl/types/Node.tl diff --git a/src/awesomewm.d.tl/types/Visitor.tl b/src/awesomewm.d.tl/types/Visitor.tl new file mode 100644 index 0000000..9b2b26e --- /dev/null +++ b/src/awesomewm.d.tl/types/Visitor.tl @@ -0,0 +1,7 @@ +local type Node = require("types.Node") + +local record Visitor + visit: function(Node) +end + +return Visitor diff --git a/src/awesomewm.d.tl/visitors/type_mapping.tl b/src/awesomewm.d.tl/visitors/type_mapping.tl new file mode 100644 index 0000000..430ec5d --- /dev/null +++ b/src/awesomewm.d.tl/visitors/type_mapping.tl @@ -0,0 +1,68 @@ +local type Node = require("types.Node") +local type Visitor = require("types.Visitor") + +-- Special types I don't want to deal with for now +local gears_shape_function = "function(cr: any, width: integer, height: integer)" + +local type_map : { string : string } = { + awesome = "awesome", + Awesome = "awesome", + bool = "boolean", + client = "Client", + ["gears.shape"] = gears_shape_function, + ["gears.surface"] = "Surface", + image = "Image", + int = "integer", + screen = "Screen", + shape = gears_shape_function, + surface = "Surface", + tag = "Tag", + ["wibox.widget"] = "Widget", + widget = "Widget", +} + +local function get_type(t: string): string + return type_map[t] or t +end + +local function check_node(node: Node) + if not node.types then + return + end + + for i, t in ipairs(node.types) do + node.types[i] = get_type(t) + end +end + +local function check_function_parameters(node: Node) + if not node.parameters then + return + end + + for _, parameter in ipairs(node.parameters) do + check_node(parameter) + end +end + +local function check_function_returns(node: Node) + if not node.return_types then + return + end + + for i, ret in ipairs(node.return_types) do + node.return_types[i] = get_type(ret) + end +end + +local record Type_Mapping + visit: function(node: Node) +end + +function Type_Mapping.visit(node: Node) + check_node(node) + check_function_parameters(node) + check_function_returns(node) +end + +return Type_Mapping From e076a08572095d1ea79aae832263d677e4d1c9d5 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 21 May 2023 20:00:09 +0200 Subject: [PATCH 29/39] feat(Visitors): implement module_dependencies --- src/awesomewm.d.tl/ast.tl | 10 +++--- src/awesomewm.d.tl/dag.tl | 20 ++--------- src/awesomewm.d.tl/init.tl | 13 +++++++ src/awesomewm.d.tl/scraper/module_doc.tl | 8 ++--- src/awesomewm.d.tl/types/Dag.tl | 2 +- src/awesomewm.d.tl/types/Node.tl | 4 +++ src/awesomewm.d.tl/types/Visitor.tl | 7 ---- src/awesomewm.d.tl/utils.tl | 9 +++-- .../visitors/module_dependencies.tl | 35 +++++++++++++++++++ src/awesomewm.d.tl/visitors/type_mapping.tl | 1 - 10 files changed, 72 insertions(+), 37 deletions(-) delete mode 100644 src/awesomewm.d.tl/types/Visitor.tl create mode 100644 src/awesomewm.d.tl/visitors/module_dependencies.tl diff --git a/src/awesomewm.d.tl/ast.tl b/src/awesomewm.d.tl/ast.tl index b74390e..6f5dae8 100644 --- a/src/awesomewm.d.tl/ast.tl +++ b/src/awesomewm.d.tl/ast.tl @@ -1,10 +1,12 @@ local type Node = require("types.Node") -local basic_nodes : { Node.Token : function(name: string): Node } = { - module = function(name: string): Node +local basic_nodes : { Node.Token : function(name: string, ...: any): Node } = { + module = function(name: string, module_path: string): Node return { token = "module", name = name, + module_path = module_path, + dependencies = {}, children = {}, } end, @@ -53,8 +55,8 @@ local basic_nodes : { Node.Token : function(name: string): Node } = { end, } -local function create_node(token: Node.Token, name: string): Node - local node = basic_nodes[token](name) +local function create_node(token: Node.Token, name: string, ...: any): Node + local node = basic_nodes[token](name, ...) return node end diff --git a/src/awesomewm.d.tl/dag.tl b/src/awesomewm.d.tl/dag.tl index b2ff5bf..d981263 100644 --- a/src/awesomewm.d.tl/dag.tl +++ b/src/awesomewm.d.tl/dag.tl @@ -1,27 +1,13 @@ local type Dag = require("types.Dag") -local type Node = require("types.Node") -local function init(): Dag +local function new(): Dag local dag : Dag = { - nodes_by_module_name = {}, + modules = {}, global_nodes = {}, } return dag end -local function insert(dag: Dag, module_name: string, node: Node) - if not dag.nodes_by_module_name[module_name] then - dag.nodes_by_module_name[module_name] = {} - end - table.insert(dag.nodes_by_module_name[module_name], node) -end - -local function insert_global(dag: Dag, node: Node) - table.insert(dag.global_nodes, node) -end - return { - init = init, - insert = insert, - insert_global = insert_global, + new = new, } diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index f1bc8c2..1d23308 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -11,6 +11,7 @@ local logger = require "logger" local Map = require "pl.Map" local Module_Doc = require "entity.Module_Doc" local Module_Info = require "entity.Module_Info" +local module_dependencies = require "visitors.module_dependencies" local type Node = require "types.Node" local property = require "property" local scraper = require "scraper" @@ -156,10 +157,22 @@ local module_ast, other_nodes = scraper.module_doc.get_doc_from_page( ast.in_order_visitor(module_ast, function(node: Node) type_mapping.visit(node) end) +ast.in_order_visitor(module_ast, function(node: Node) + module_dependencies.visit(node, module_ast, { + modules = { + Screen = { + name = "screen", + module_path = "awful.screen", + token = "module", + }, + } + }) +end) log:info(logger.message_with_metadata("Finished", { module_ast = module_ast, other_nodes = other_nodes, + deps = module_ast.dependencies, })) filesystem.file_writer.write( diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index 9ef4c64..ba5663c 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -245,7 +245,7 @@ local section_scrapers : { Section : function(html: string, record_name: local module = {} -function module.get_doc_from_page(html: string, module_name: string): Node, { Node } +function module.get_doc_from_page(html: string, module_path: string): Node, { Node } local html_nodes = scraper_utils.extract_nodes(html, { "h2.section-header", "dl.function", @@ -255,8 +255,8 @@ function module.get_doc_from_page(html: string, module_name: string): Node, { No error "The list aren't the same size!" end - local record_name = utils.capitalize((module_name:gsub(".*%.", ""))) - local module_root = ast.create_node("module", record_name) + local record_name = utils.capitalize((module_path:gsub(".*%.", ""))) + local module_root = ast.create_node("module", record_name, module_path) local other_nodes : { Node } = {} local module_signals_node = ast.create_node("enum", "Signal") @@ -268,7 +268,7 @@ function module.get_doc_from_page(html: string, module_name: string): Node, { No local dl_html = html_nodes:get("dl.function")[i]:outer_html() if section_scrapers[section_name] then - local module_nodes, global_nodes, signals_name = section_scrapers[section_name](dl_html, record_name, module_name) + local module_nodes, global_nodes, signals_name = section_scrapers[section_name](dl_html, record_name, module_path) for _, node in ipairs(module_nodes) do table.insert(module_root.children, node) end diff --git a/src/awesomewm.d.tl/types/Dag.tl b/src/awesomewm.d.tl/types/Dag.tl index 481ffa2..ad81e3e 100644 --- a/src/awesomewm.d.tl/types/Dag.tl +++ b/src/awesomewm.d.tl/types/Dag.tl @@ -1,7 +1,7 @@ local type Node = require("types.Node") local record Dag - nodes_by_module_name: { string : { Node } } + modules: { string : Node } -- module_name -> root_node (token = "module") global_nodes: { Node } end diff --git a/src/awesomewm.d.tl/types/Node.tl b/src/awesomewm.d.tl/types/Node.tl index b0440aa..fa841cc 100644 --- a/src/awesomewm.d.tl/types/Node.tl +++ b/src/awesomewm.d.tl/types/Node.tl @@ -20,6 +20,10 @@ local record Node -- for "function" and "metamethod" parameters: { Node } return_types: { string } + + -- for "module" + module_path: string + dependencies: { string : string } -- module_name -> module_path end return Node diff --git a/src/awesomewm.d.tl/types/Visitor.tl b/src/awesomewm.d.tl/types/Visitor.tl deleted file mode 100644 index 9b2b26e..0000000 --- a/src/awesomewm.d.tl/types/Visitor.tl +++ /dev/null @@ -1,7 +0,0 @@ -local type Node = require("types.Node") - -local record Visitor - visit: function(Node) -end - -return Visitor diff --git a/src/awesomewm.d.tl/utils.tl b/src/awesomewm.d.tl/utils.tl index 6c695bc..360dff5 100644 --- a/src/awesomewm.d.tl/utils.tl +++ b/src/awesomewm.d.tl/utils.tl @@ -76,10 +76,13 @@ function utils.do_or_fail(func: function(...: any): (T | nil, string), ... return res as T -- promote to T since pcall succeeded at this point end -function utils.spread(t: { T }, i: { T }) - for _, v in ipairs(i) do - table.insert(t, v) +function utils.spread(t: { T }, ...: { T }): { T } + for _, a in ipairs({ ... }) do + for _, v in ipairs(a) do + table.insert(t, v) + end end + return t end return utils diff --git a/src/awesomewm.d.tl/visitors/module_dependencies.tl b/src/awesomewm.d.tl/visitors/module_dependencies.tl new file mode 100644 index 0000000..977ef1b --- /dev/null +++ b/src/awesomewm.d.tl/visitors/module_dependencies.tl @@ -0,0 +1,35 @@ +local type Dag = require("types.Dag") +local type Node = require("types.Node") +local utils = require("utils") + +local spread = utils.spread + +local function get_all_types_in_node(node: Node): { string } + local parameters_types = {} + if node.parameters then + for _, v in ipairs(node.parameters) do + spread(parameters_types, v.types) + end + end + return spread( + {}, + node.types or {}, + node.return_types or {}, + parameters_types or {}) +end + +local record Module_Dependencies + visit: function(node: Node, mod: Node, d: Dag) +end + +function Module_Dependencies.visit(node: Node, mod: Node, d: Dag) + local all_types = get_all_types_in_node(node) + for _, v in ipairs(all_types) do + local dependency = d.modules[v] + if dependency then + mod.dependencies[dependency.name] = dependency.module_path + end + end +end + +return Module_Dependencies diff --git a/src/awesomewm.d.tl/visitors/type_mapping.tl b/src/awesomewm.d.tl/visitors/type_mapping.tl index 430ec5d..b3d8933 100644 --- a/src/awesomewm.d.tl/visitors/type_mapping.tl +++ b/src/awesomewm.d.tl/visitors/type_mapping.tl @@ -1,5 +1,4 @@ local type Node = require("types.Node") -local type Visitor = require("types.Visitor") -- Special types I don't want to deal with for now local gears_shape_function = "function(cr: any, width: integer, height: integer)" From 4d81aaef56745fe9dd86cdc13617ae8708788cee Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 21 May 2023 20:17:37 +0200 Subject: [PATCH 30/39] feat(printer): render `require` statements --- src/awesomewm.d.tl/init.tl | 2 +- src/awesomewm.d.tl/printer/teal_type_definition.tl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index 1d23308..2cecf43 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -161,7 +161,7 @@ ast.in_order_visitor(module_ast, function(node: Node) module_dependencies.visit(node, module_ast, { modules = { Screen = { - name = "screen", + name = "Screen", module_path = "awful.screen", token = "module", }, diff --git a/src/awesomewm.d.tl/printer/teal_type_definition.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl index 4f089e4..05724f2 100644 --- a/src/awesomewm.d.tl/printer/teal_type_definition.tl +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -28,6 +28,14 @@ local function render_code(code: string, indent_level: integer): string return generated end +local function render_require(dependencies: { string : string }): string + local generated = "" + for dependency, path in pairs(dependencies) do + generated = generated .. string.format("local %s = require(\"%s\")\n", dependency, path) + end + return generated:sub(1, -2) -- remove the last newline +end + local record Node_Printer_Function on_node: function(node: Node, indent_level: integer): string, integer before_node: nil | function(node: Node, indent_level: integer): string, integer @@ -45,9 +53,11 @@ local node_printer : { Node.Token : Node_Printer_Function } = { string.format( [[ -- This file was auto-generated. + %s local record %s ]], + render_require(node.dependencies), node.name), indent_level), indent_level + 1 end, From bea2f0fc54329fa4227ac1652948f5371b4eea36 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 21 May 2023 20:42:32 +0200 Subject: [PATCH 31/39] spec(module_doc): fix and refactor Fix the tests by adding the new keys from `"module"` tokens. We now use a `test` function to make the `it` easier to write/read. --- spec/scraper/module_doc_spec.tl | 466 +++++++++++++++++--------------- 1 file changed, 242 insertions(+), 224 deletions(-) diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl index b56b53d..d563fd4 100644 --- a/spec/scraper/module_doc_spec.tl +++ b/spec/scraper/module_doc_spec.tl @@ -4,10 +4,19 @@ local scraper = require("scraper.module_doc") local get_doc_from_page = scraper.get_doc_from_page +local function test(html: string, module_path: string, expected_ast: Node, expected_other_nodes: { Node } | nil): function() + return function() + local ast , other_nodes = get_doc_from_page(html, module_path) + assert.same(expected_ast, ast) + assert.same(expected_other_nodes or {}, other_nodes) + end +end + describe("Scrap documentation", function() - it("should return a valid AST for an empty module", function() - local ast , nodes = get_doc_from_page("", "empty") - local expected : Node = { + it("should return a valid AST for an empty module", test( + "", + "empty", + { children = { { children = {}, @@ -16,50 +25,50 @@ describe("Scrap documentation", function() } }, name = "Empty", + module_path = "empty", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - assert.same({}, nodes) - end) + })) - it("should produce Variable and `property::` Signal nodes", function() - local ast = get_doc_from_page([[ -

- Object properties -

-
-
- 馃敆 - value - number - 路 1 signal -
-
-

Constraints:

- - - - - - - - - - -
- - Default value - - : 0
- - Negative allowed - - : true
- -
-
- ]], "property_signal") - local expected : Node = { + it("should produce Variable and `property::` Signal nodes", test( + [[ +

+ Object properties +

+
+
+ 馃敆 + value + number + 路 1 signal +
+
+

Constraints:

+ + + + + + + + + + +
+ + Default value + + : 0
+ + Negative allowed + + : true
+ +
+
+ ]], + "property_signal", + { children = { { children = { @@ -78,82 +87,83 @@ describe("Scrap documentation", function() } }, name = "Property_signal", + module_path = "property_signal", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - end) + })) - it("should produce Enum nodes when an Object Property type is a String with constraints", function() - local ast = get_doc_from_page([[ + it("should produce Enum nodes when an Object Property type is a String with constraints", test( + [[

Object properties

-
- 馃敆 - horizontal_fit_policy - string - 路 1 signal -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- Default value - : "auto"
- Valid values: -
- - "auto" - - - : Honor the resize variable and preserve the aspect - ratio. -
- - "none" - - : Do not resize at all.
- - "fit" - - : Resize to the widget width.
- -
+
+ 馃敆 + horizontal_fit_policy + string + 路 1 signal +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Default value + : "auto"
+ Valid values: +
+ + "auto" + + + : Honor the resize variable and preserve the aspect + ratio. +
+ + "none" + + : Do not resize at all.
+ + "fit" + + : Resize to the widget width.
+ +
- ]], "property_enum") - local expected : Node = { + ]], + "property_enum", + { children = { { children = { @@ -190,29 +200,30 @@ describe("Scrap documentation", function() }, }, name = "Property_enum", + module_path = "property_enum", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - end) + })) - it("should produce a `string` typed Variable node when a String Property has no constraint", function() - local ast = get_doc_from_page([[ -

- Object properties -

-
-
- 馃敆 - markup - string - 路 1 signal -
-
- string -
-
- ]], "property_string") - local expected : Node = { + it("should produce a `string` typed Variable node when a String Property has no constraint", test( + [[ +

+ Object properties +

+
+
+ 馃敆 + markup + string + 路 1 signal +
+
+ string +
+
+ ]], + "property_string", + { children = { { children = { @@ -231,47 +242,48 @@ describe("Scrap documentation", function() } }, name = "Property_string", + module_path = "property_string", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - end) + })) - it("should provide a Function node with the `self` as the first positional parameter", function() - local ast = get_doc_from_page([[ -

Object methods

-
-
- 馃敆 - :swap (tag2) - - -
-
-

Parameters:

- - - - - - - - - - - - - - - -
NameType(s)Description
tag2 - tag - The second tag
-
-
- ]], "awful.tag") - local expected : Node = { + it("should provide a Function node with the `self` as the first positional parameter", test( + [[ +

Object methods

+
+
+ 馃敆 + :swap (tag2) + + +
+
+

Parameters:

+ + + + + + + + + + + + + + + +
NameType(s)Description
tag2 + tag + The second tag
+
+
+ ]], + "awful.tag", + { children = { { children = {}, @@ -297,15 +309,15 @@ describe("Scrap documentation", function() }, }, name = "Tag", + module_path = "awful.tag", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - end) + })) - it("should produce Signal nodes", function() - local ast = get_doc_from_page([[ -

Signals

-
+ it("should produce Signal nodes", test( + [[ +

Signals

+
路 Inherited from wibox.widget.base
-
- ]], "signal") - local expected : Node = { +
+ ]], + "signal", + { children = { { children = { @@ -348,13 +361,13 @@ describe("Scrap documentation", function() }, }, name = "Signal", + module_path = "signal", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - end) + })) - it("should produce Function nodes", function() - local ast = get_doc_from_page([[ + it("should produce Function nodes", test( + [[

Static module functions

@@ -415,8 +428,9 @@ describe("Scrap documentation", function() - ]], "awesome") -- The module name must be the same as the module name in the doc - local expected : Node = { + ]], + "awesome", -- The module name must be the same as the module name in the doc + { children = { { children = {}, @@ -442,13 +456,13 @@ describe("Scrap documentation", function() } }, name = "Awesome", + module_path = "awesome", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - end) + })) - it("should produce a Record node when a function parameter is a named-parameter-table", function() - local ast = get_doc_from_page([[ + it("should produce a Record node when a function parameter is a named-parameter-table", test( + [[

Static module functions

@@ -529,8 +543,9 @@ describe("Scrap documentation", function() - ]], "awful.screen") - assert.same(ast, { + ]], + "awful.screen", + { children = { { children = {}, @@ -567,12 +582,13 @@ describe("Scrap documentation", function() }, }, name = "Screen", + module_path = "awful.screen", + dependencies = {}, token = "module", - }) - end) + })) - it("should go back to a table typed parameter when the record is empty", function() - local ast = get_doc_from_page([[ + it("should go back to a table typed parameter when the record is empty", test( + [[

Static module functions

@@ -624,8 +640,9 @@ describe("Scrap documentation", function()
- ]], "gears.table") - local expected : Node = { + ]], + "gears.table", + { children = { { children = {}, @@ -656,13 +673,13 @@ describe("Scrap documentation", function() } }, name = "Table", + module_path = "gears.table", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - end) + })) - it("should go back to a table typed parameter when the record is empty and it's the last parameter", function() - local ast = get_doc_from_page([[ + it("should go back to a table typed parameter when the record is empty and it's the last parameter", test( + [[

Object methods

@@ -720,8 +737,9 @@ describe("Scrap documentation", function()
- ]], "awful.client") - local expected : Node = { + ]], + "awful.client", + { children = { { children = {}, @@ -747,13 +765,13 @@ describe("Scrap documentation", function() }, }, name = "Client", + module_path = "awful.client", + dependencies = {}, token = "module", - } - assert.same(expected, ast) - end) + })) - it("should return Function nodes with the `other_nodes` list when the function module name doesn't match the module name", function() - local ast , other_nodes = get_doc_from_page([[ + it("should return Function nodes with the `other_nodes` list when the function module name doesn't match the module name", test( + [[

Static module functions

@@ -779,8 +797,9 @@ describe("Scrap documentation", function() - ]], "awful.client") - local expected_ast : Node = { + ]], + "awful.client", + { children = { { children = {}, @@ -789,17 +808,16 @@ describe("Scrap documentation", function() }, }, name = "Client", + module_path = "awful.client", + dependencies = {}, token = "module", - } - assert.same(expected_ast, ast) - local expected_other_nodes : { Node } = { + }, + { { parameters = {}, return_types = { "integer" }, name = "client.instances", token = "function", } - } - assert.same(expected_other_nodes, other_nodes) - end) + })) end) From 8d54ab47464bd4d024e74b35e8c1b8ab32a60d4c Mon Sep 17 00:00:00 2001 From: Aire-One Date: Tue, 18 Jul 2023 20:25:01 +0200 Subject: [PATCH 32/39] fix(spec): ASTs need to have `dependencies` --- spec/printer/teal_type_definition_spec.tl | 5 +++++ src/awesomewm.d.tl/printer/teal_type_definition.tl | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl index 1174c1e..913df2c 100644 --- a/spec/printer/teal_type_definition_spec.tl +++ b/spec/printer/teal_type_definition_spec.tl @@ -21,6 +21,7 @@ describe("Teal type definition Printer", function() it("should print a simple module type definition", gen( { children = {}, + dependencies = {}, name = "Empty", token = "module", }, @@ -51,6 +52,7 @@ describe("Teal type definition Printer", function() token = "enum", }, }, + dependencies = {}, name = "Signal_Module", token = "module", }, @@ -76,6 +78,7 @@ describe("Teal type definition Printer", function() token = "variable", } }, + dependencies = {}, name = "Property_Module", token = "module", }, @@ -110,6 +113,7 @@ describe("Teal type definition Printer", function() token = "function", }, }, + dependencies = {}, name = "Function_Module", token = "module", }, @@ -132,6 +136,7 @@ describe("Teal type definition Printer", function() token = "record", } }, + dependencies = {}, name = "Nested_Module", token = "module", }, diff --git a/src/awesomewm.d.tl/printer/teal_type_definition.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl index 05724f2..1c6fefe 100644 --- a/src/awesomewm.d.tl/printer/teal_type_definition.tl +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -54,10 +54,9 @@ local node_printer : { Node.Token : Node_Printer_Function } = { [[ -- This file was auto-generated. %s - local record %s ]], - render_require(node.dependencies), + render_require(node.dependencies), -- last require statement will have a newline node.name), indent_level), indent_level + 1 end, From 4f247253515be25427f5d3c9dbf00b90d9e77b86 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Fri, 28 Jul 2023 01:10:41 +0200 Subject: [PATCH 33/39] spec(teal_type_definition): test module dependencies --- spec/printer/teal_type_definition_spec.tl | 21 +++++++++++++++++++ .../printer/teal_type_definition.tl | 11 ++++------ src/awesomewm.d.tl/utils.tl | 16 ++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl index 913df2c..005b2fd 100644 --- a/spec/printer/teal_type_definition_spec.tl +++ b/spec/printer/teal_type_definition_spec.tl @@ -150,4 +150,25 @@ describe("Teal type definition Printer", function() return Nested_Module ]])) + + it("should print require statement for module's dependencies", gen( + { + children = {}, + dependencies = { + dep = "dep", + deeper = "path.dep.deeper", + }, + name = "Module", + token = "module", + }, + [[ + -- This file was auto-generated. + local deeper = require("path.dep.deeper") + local dep = require("dep") + + local record Module + end + + return Module + ]])) end) diff --git a/src/awesomewm.d.tl/printer/teal_type_definition.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl index 1c6fefe..7dce917 100644 --- a/src/awesomewm.d.tl/printer/teal_type_definition.tl +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -2,6 +2,7 @@ local ast = require("ast") local logger = require("logger") local type Node = require("types.Node") local stringx = require("pl.stringx") +local utils = require("utils") local log = logger.log("scraper") @@ -30,10 +31,10 @@ end local function render_require(dependencies: { string : string }): string local generated = "" - for dependency, path in pairs(dependencies) do + for dependency, path in utils.pairsByKeys(dependencies) do generated = generated .. string.format("local %s = require(\"%s\")\n", dependency, path) end - return generated:sub(1, -2) -- remove the last newline + return generated end local record Node_Printer_Function @@ -51,11 +52,7 @@ local node_printer : { Node.Token : Node_Printer_Function } = { before_node = function(node: Node, indent_level: integer): string, integer return render_code( string.format( - [[ - -- This file was auto-generated. - %s - local record %s - ]], + "-- This file was auto-generated.\n%s\nlocal record %s", render_require(node.dependencies), -- last require statement will have a newline node.name), indent_level), indent_level + 1 diff --git a/src/awesomewm.d.tl/utils.tl b/src/awesomewm.d.tl/utils.tl index 360dff5..d1e1019 100644 --- a/src/awesomewm.d.tl/utils.tl +++ b/src/awesomewm.d.tl/utils.tl @@ -85,4 +85,20 @@ function utils.spread(t: { T }, ...: { T }): { T } return t end +function utils.pairsByKeys(list: { Key : Value }, comp: function(Key, Key): boolean): function(): Key, Value + local sortedKeys = {} + for n in pairs(list) do table.insert(sortedKeys, n) end + table.sort(sortedKeys, comp) + + local index = 0 + return function(): Key, Value + index = index + 1 + if sortedKeys[index] == nil then + return nil + end + + return sortedKeys[index], list[sortedKeys[index]] + end +end + return utils From 6441a6a50bc29db0679e06b524a49691152d08e9 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Mon, 31 Jul 2023 19:02:05 +0200 Subject: [PATCH 34/39] fix(visitor): module_dependencies renames dependency usages --- .../visitors/module_dependencies.tl | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/awesomewm.d.tl/visitors/module_dependencies.tl b/src/awesomewm.d.tl/visitors/module_dependencies.tl index 977ef1b..844744e 100644 --- a/src/awesomewm.d.tl/visitors/module_dependencies.tl +++ b/src/awesomewm.d.tl/visitors/module_dependencies.tl @@ -18,16 +18,45 @@ local function get_all_types_in_node(node: Node): { string } parameters_types or {}) end +local function replace_in_node_type(node: Node, old_type: string, new_type: string) + if node.parameters then + for _, v in ipairs(node.parameters) do + for i, t in ipairs(v.types) do + if t == old_type then + v.types[i] = new_type + end + end + end + end + + if node.types then + for i, t in ipairs(node.types) do + if t == old_type then + node.types[i] = new_type + end + end + end + + if node.return_types then + for i, t in ipairs(node.return_types) do + if t == old_type then + node.return_types[i] = new_type + end + end + end +end + local record Module_Dependencies visit: function(node: Node, mod: Node, d: Dag) end function Module_Dependencies.visit(node: Node, mod: Node, d: Dag) local all_types = get_all_types_in_node(node) - for _, v in ipairs(all_types) do - local dependency = d.modules[v] + for _, type_name in ipairs(all_types) do + local dependency = d.modules[type_name] or d.modules[utils.lowercase(type_name)] if dependency then mod.dependencies[dependency.name] = dependency.module_path + replace_in_node_type(node, dependency.module_path, dependency.name) end end end From 284b49898c05c5c5786ac7fb444ed8c425c5fef9 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Mon, 31 Jul 2023 19:02:40 +0200 Subject: [PATCH 35/39] fix(visitor): module_dependencies shouldn't self require --- src/awesomewm.d.tl/visitors/module_dependencies.tl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/awesomewm.d.tl/visitors/module_dependencies.tl b/src/awesomewm.d.tl/visitors/module_dependencies.tl index 844744e..94cc701 100644 --- a/src/awesomewm.d.tl/visitors/module_dependencies.tl +++ b/src/awesomewm.d.tl/visitors/module_dependencies.tl @@ -53,11 +53,17 @@ end function Module_Dependencies.visit(node: Node, mod: Node, d: Dag) local all_types = get_all_types_in_node(node) for _, type_name in ipairs(all_types) do + if type_name == mod.name then + goto continue + end + local dependency = d.modules[type_name] or d.modules[utils.lowercase(type_name)] if dependency then mod.dependencies[dependency.name] = dependency.module_path replace_in_node_type(node, dependency.module_path, dependency.name) end + + ::continue:: end end From f44a02868392e3db7e927e6efceea9180cf11642 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Mon, 31 Jul 2023 19:20:32 +0200 Subject: [PATCH 36/39] fix(scraper): rename constructor methods to `new` --- src/awesomewm.d.tl/scraper/module_doc.tl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index ba5663c..0eb0f66 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -212,7 +212,13 @@ end -- - Strings that should be added to the record Signals local section_scrapers : { Section : function(html: string, record_name: string, module_name: string): { Node }, { Node }, { string } } = { ["Constructors"] = function(html: string): { Node }, { Node }, { string } - return extract_section_functions(html), {}, {} + local constructors = extract_section_functions(html) + for _, constructor in ipairs(constructors) do + if constructor.token == "function" then + constructor.name = "new" + end + end + return constructors, {}, {} end, ["Static module functions"] = function(html: string, _: string, module_name: string): { Node }, { Node }, { string } local static_functions, other_functions = extract_section_functions(html, module_name) From dbeb53639390e5a86478bf37bc0f11c9a54a343a Mon Sep 17 00:00:00 2001 From: Aire-One Date: Mon, 31 Jul 2023 19:26:19 +0200 Subject: [PATCH 37/39] feat(printer): teal_type_definition should use the `require type` notation --- spec/printer/teal_type_definition_spec.tl | 4 ++-- src/awesomewm.d.tl/printer/teal_type_definition.tl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl index 005b2fd..f6367ac 100644 --- a/spec/printer/teal_type_definition_spec.tl +++ b/spec/printer/teal_type_definition_spec.tl @@ -163,8 +163,8 @@ describe("Teal type definition Printer", function() }, [[ -- This file was auto-generated. - local deeper = require("path.dep.deeper") - local dep = require("dep") + local type deeper = require("path.dep.deeper") + local type dep = require("dep") local record Module end diff --git a/src/awesomewm.d.tl/printer/teal_type_definition.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl index 7dce917..fbfab82 100644 --- a/src/awesomewm.d.tl/printer/teal_type_definition.tl +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -32,7 +32,7 @@ end local function render_require(dependencies: { string : string }): string local generated = "" for dependency, path in utils.pairsByKeys(dependencies) do - generated = generated .. string.format("local %s = require(\"%s\")\n", dependency, path) + generated = generated .. string.format("local type %s = require(\"%s\")\n", dependency, path) end return generated end From 2506ce9d585c370880d366a14e8229558bcfb407 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Tue, 1 Aug 2023 23:22:45 +0200 Subject: [PATCH 38/39] run: let's make the main entry point do something --- src/awesomewm.d.tl/dag.tl | 17 +++ src/awesomewm.d.tl/init.tl | 186 +++++++++++++++----------------- src/awesomewm.d.tl/types/Dag.tl | 2 +- 3 files changed, 102 insertions(+), 103 deletions(-) diff --git a/src/awesomewm.d.tl/dag.tl b/src/awesomewm.d.tl/dag.tl index d981263..9953a87 100644 --- a/src/awesomewm.d.tl/dag.tl +++ b/src/awesomewm.d.tl/dag.tl @@ -1,4 +1,6 @@ local type Dag = require("types.Dag") +local type Node = require("types.Node") +local utils = require("utils") local function new(): Dag local dag : Dag = { @@ -8,6 +10,21 @@ local function new(): Dag return dag end +local function push_module(dag: Dag, module_path: string, root_node: Node) + dag.modules[module_path] = root_node +end + +local function push_global_nodes(dag: Dag, nodes: { Node }) + utils.spread(dag.global_nodes, nodes) +end + +local function iter_modules(dag: Dag): function(): string, Node + return pairs(dag.modules) +end + return { new = new, + push_module = push_module, + push_global_nodes = push_global_nodes, + iter_modules = iter_modules, } diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index 2cecf43..fd99f4c 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -4,12 +4,12 @@ end local ast = require "ast" local crawler = require "crawler" +local dag = require "dag" local filesystem = require "filesystem" local printer = require "printer" local List = require "pl.List" local logger = require "logger" local Map = require "pl.Map" -local Module_Doc = require "entity.Module_Doc" local Module_Info = require "entity.Module_Info" local module_dependencies = require "visitors.module_dependencies" local type Node = require "types.Node" @@ -73,109 +73,91 @@ local function modules_tree(modules: List): Map +) + local tree = modules_tree(module_infos) + for module, children in tree:iter() do + -- TODO : this map should be coupled with the all_module_infos list + local requires: Map = Map() + for child in children:iter() do + local name = child:gmatch(".*%.(.*)$")() + requires:set(name, child) + end + filesystem.file_writer.write( + printer.module_init_definition.generate_teal(requires), + property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl" + ) + end +end --- local all_module_infos, module_infos, global_module_infos = module_lists( --- property.base_url .. property.index_uri, --- List(property.capi_modules), --- List(property.ignored_modules) --- ) --- local tree = modules_tree(module_infos) - --- log:info( --- logger.message_with_metadata( --- "Finished Module List scrapping", --- { --- total_module_count = #all_module_infos, --- module_count = #module_infos, --- global_module_count = #global_module_infos, --- tree_items = tree:len(), --- } --- ) --- ) - --- for module in module_infos:iter() do --- do_one_file( --- property.base_url .. "/" .. module.uri, --- module.name, --- property.out_directory .. "/" .. module.name:gsub("%.", "/") .. ".d.tl" --- ) --- end - --- local global_env_def: List = List() --- for module in global_module_infos:iter() do --- if module.name:gmatch(".*%sand%s.*") then --- do_one_file( --- property.base_url .. "/" .. module.uri, --- module.name, --- property.out_directory .. "/" .. module.name:gsub(".*%sand%s", ""):gsub("%.", "/") .. ".d.tl" --- ) --- end - --- local html = crawler.fetch(property.base_url .. "/" .. module.uri) --- local module_doc = scraper.module_doc.get_doc_from_page(html, (module.name:gsub("%sand%s.*", ""))) --- module_doc:fixup() --- module_doc.record_name = utils.lowercase(module_doc.record_name) --- global_env_def:append(module_doc) --- end --- filesystem.file_writer.write( --- printer.global_env_def.generate_teal(global_env_def), --- property.out_directory .. "/global_env.d.tl" --- ) - --- for module, children in tree:iter() do --- -- TODO : this map should be coupled with the all_module_infos list --- local requires: Map = Map() --- for child in children:iter() do --- local name = child:gmatch(".*%.(.*)$")() --- requires:set(name, child) --- end --- filesystem.file_writer.write( --- printer.module_init_definition.generate_teal(requires), --- property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl" --- ) --- end - -local module_ast, other_nodes = scraper.module_doc.get_doc_from_page( - crawler.fetch(property.base_url .. "/core_components/tag.html"), - "awful.tag" +--- TODO : rewrite the module_info thingy +local all_module_infos, module_infos, global_module_infos = module_lists( + property.base_url .. property.index_uri, + List(),-- List(property.capi_modules), + List(utils.spread( + property.ignored_modules, + { + -- Modules that broke the parser. + --- TODO : fix the parser + "awful.screenshot", + })) ) -ast.in_order_visitor(module_ast, function(node: Node) - type_mapping.visit(node) -end) -ast.in_order_visitor(module_ast, function(node: Node) - module_dependencies.visit(node, module_ast, { - modules = { - Screen = { - name = "Screen", - module_path = "awful.screen", - token = "module", - }, +log:info( + logger.message_with_metadata( + "Finished Module List scrapping", + { + total_module_count = #all_module_infos, + module_count = #module_infos, + global_module_count = #global_module_infos, } - }) -end) - -log:info(logger.message_with_metadata("Finished", { - module_ast = module_ast, - other_nodes = other_nodes, - deps = module_ast.dependencies, -})) - -filesystem.file_writer.write( - printer.teal_type_definition.printer(module_ast), - property.out_directory .. "/" .. "generated_from_ast" .. ".d.tl" + ) ) + +-- Build the DAG +local module_dag = dag.new() +for module in module_infos:iter() do + local module_ast, other_nodes = scraper.module_doc.get_doc_from_page( + crawler.fetch(property.base_url .. "/" .. module.uri), + (module.name:gsub(".*%sand%s", "")) + ) + + dag.push_module(module_dag, module_ast.module_path, module_ast) + dag.push_global_nodes(module_dag, other_nodes) +end + +-- Run the visitors +for _,root in dag.iter_modules(module_dag) do + ast.in_order_visitor(root, function(node: Node) + type_mapping.visit(node) + end) + ast.in_order_visitor(root, function(node: Node) + module_dependencies.visit(node, root, module_dag) + end) +end + +-- Build the global module from dag.global_nodes +--- TODO : todo + +--- TODO : this is fun, but we need to do something with it +-- Write the DAG to a file +-- local inspect = require("inspect") +-- filesystem.file_writer.write( +-- inspect(module_dag, { newline = "\n", indent = " ", depth = 2 }), +-- "generated_dag.lua" +-- ) + +log:info("Preprocessing finished") + +-- Write modules types definitions to files +for module_path, root in dag.iter_modules(module_dag) do + filesystem.file_writer.write( + printer.teal_type_definition.printer(root), + property.out_directory .. "/" .. module_path:gsub("%.", "/") .. ".d.tl" + ) +end + +do_module_init_definition(module_infos) +log:info("Module init files generated") diff --git a/src/awesomewm.d.tl/types/Dag.tl b/src/awesomewm.d.tl/types/Dag.tl index ad81e3e..0f52f13 100644 --- a/src/awesomewm.d.tl/types/Dag.tl +++ b/src/awesomewm.d.tl/types/Dag.tl @@ -1,7 +1,7 @@ local type Node = require("types.Node") local record Dag - modules: { string : Node } -- module_name -> root_node (token = "module") + modules: { string : Node } -- module_path (AKA "full name" `package.module.name`) -> root_node (token = "module") global_nodes: { Node } end From 73f6074ef2849db6b751964eefbc10a760842ed2 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Tue, 1 Aug 2023 23:36:47 +0200 Subject: [PATCH 39/39] fix(visitor/type_mapping): update mapping --- src/awesomewm.d.tl/visitors/type_mapping.tl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/awesomewm.d.tl/visitors/type_mapping.tl b/src/awesomewm.d.tl/visitors/type_mapping.tl index b3d8933..9c6e74a 100644 --- a/src/awesomewm.d.tl/visitors/type_mapping.tl +++ b/src/awesomewm.d.tl/visitors/type_mapping.tl @@ -4,8 +4,7 @@ local type Node = require("types.Node") local gears_shape_function = "function(cr: any, width: integer, height: integer)" local type_map : { string : string } = { - awesome = "awesome", - Awesome = "awesome", + awesome = "Awesome", bool = "boolean", client = "Client", ["gears.shape"] = gears_shape_function, @@ -16,8 +15,10 @@ local type_map : { string : string } = { shape = gears_shape_function, surface = "Surface", tag = "Tag", - ["wibox.widget"] = "Widget", - widget = "Widget", + widget = "wibox.widget", + + -- fixes we shouldn't have to do (We need to PR Awesome to fix the doc) + timer = "gears.timer", } local function get_type(t: string): string