From 0bea4723b577da3eb0ec54b3da4151e8d1b3b9cc Mon Sep 17 00:00:00 2001 From: Aire-One Date: Fri, 4 Nov 2022 19:46:10 +0100 Subject: [PATCH 1/7] feat(scraper): detect function parameters table --- src/awesomewm.d.tl/scraper/module_doc.tl | 45 ++++++++++++++++++------ 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index e92e23c..e139869 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -27,19 +27,44 @@ local function extract_item_name(item_name_node: scan.HTMLNode): string end local function extract_function_parameters(function_parameters_node: scan.HTMLNode): { Function_Info.Parameter } - local query_selectors = { - name = "span.parameter", - types = "span.types" - } + local current_record_parameter: string = nil - return scraper_utils.scrape_tuples( + return scraper_utils.scrape( function_parameters_node:outer_html(), - { query_selectors.name, query_selectors.types }, - function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Parameter - return { - name = extract_node_text(nodes[query_selectors.name]), - types = parse_parameter_types(extract_node_text(nodes[query_selectors.types])), + "tr", + function(line_node: scan.HTMLNode): Function_Info.Parameter + if line_node.attr ~= nil and line_node.attr.class == "see_also_sublist" then + log:debug("subtable found, skipping") + return nil + end + + local query_selectors = { + name = "span.parameter", + types = "span.types" } + + return scraper_utils.scrape_tuples( + line_node:outer_html(), + { query_selectors.name, query_selectors.types }, + function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Parameter + local name = extract_node_text(nodes[query_selectors.name] as scan.HTMLNode) + local types = parse_parameter_types(extract_node_text(nodes[query_selectors.types] as scan.HTMLNode)) + + if types == List({ "table" }) then + local record_name = utils.capitalize(name) + current_record_parameter = record_name + return { + name = name, + types = List({ record_name }), + } + end + + return { + name = name, + types = types, + } + end + )[1] end ) end From aa42419a54c8bbb6bcc57e0dbf39477f52c3aee5 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Fri, 4 Nov 2022 20:41:21 +0100 Subject: [PATCH 2/7] feat(scraper): introduce Type_Info --- src/awesomewm.d.tl/entity/Function_Info.tl | 5 ++-- src/awesomewm.d.tl/entity/Module_Doc.tl | 5 +++- src/awesomewm.d.tl/entity/Type_Info.tl | 31 ++++++++++++++++++++++ src/awesomewm.d.tl/entity/Variable_Info.tl | 3 ++- src/awesomewm.d.tl/generator/snippets.tl | 8 +++--- src/awesomewm.d.tl/scraper/module_doc.tl | 30 ++++++++++++++------- 6 files changed, 65 insertions(+), 17 deletions(-) create mode 100644 src/awesomewm.d.tl/entity/Type_Info.tl diff --git a/src/awesomewm.d.tl/entity/Function_Info.tl b/src/awesomewm.d.tl/entity/Function_Info.tl index f856539..6b8c821 100644 --- a/src/awesomewm.d.tl/entity/Function_Info.tl +++ b/src/awesomewm.d.tl/entity/Function_Info.tl @@ -1,4 +1,5 @@ local List = require "pl.List" +local Type_Info = require "entity.Type_Info" local record Function_Info metamethod __call: function(Function_Info): Function_Info @@ -6,7 +7,7 @@ local record Function_Info Function_Info: Function_Info record Parameter name: string - types: List + types: List end name: string @@ -27,7 +28,7 @@ local __Function_Info: metatable = { end, } -function Function_Info:append_parameter(name: string, types: List) +function Function_Info:append_parameter(name: string, types: List) self.parameters:append { name = name, types = types, diff --git a/src/awesomewm.d.tl/entity/Module_Doc.tl b/src/awesomewm.d.tl/entity/Module_Doc.tl index bb50903..484bd75 100644 --- a/src/awesomewm.d.tl/entity/Module_Doc.tl +++ b/src/awesomewm.d.tl/entity/Module_Doc.tl @@ -9,7 +9,9 @@ local record Module_Doc record_name: string - constructors: List + constructors: List -- Translates to a list of methods + constructor_param_record: List -- Constructor with the __call metamethod and named parameters pattern + methods: List properties: List static_functions: List @@ -20,6 +22,7 @@ local __Module_Doc: metatable = { __call = function(_: Module_Doc): Module_Doc return { constructors = List(), + constructor_param_record = List(), methods = List(), properties = List(), static_functions = List(), diff --git a/src/awesomewm.d.tl/entity/Type_Info.tl b/src/awesomewm.d.tl/entity/Type_Info.tl new file mode 100644 index 0000000..d7eb14d --- /dev/null +++ b/src/awesomewm.d.tl/entity/Type_Info.tl @@ -0,0 +1,31 @@ +local Map = require "pl.Map" + +local record Type_Info + metamethod __call: function(Type_Info, record_name: string): Type_Info + + Type_Info: Type_Info + + name: string + + -- Map : name -> type + -- We can't use Variable_Info here because it's a circular dependency. + record_entries: Map | nil +end + +local __Type_Info: metatable = { + __call = function(_self: Type_Info, record_name: string): Type_Info + if record_name ~= nil then + return { + name = record_name, + record_entries = Map() + } + end + return { + name = "", + record_entries = nil, + } + end, +} + +return setmetatable({} as Type_Info, __Type_Info) + diff --git a/src/awesomewm.d.tl/entity/Variable_Info.tl b/src/awesomewm.d.tl/entity/Variable_Info.tl index f376458..48e4ce0 100644 --- a/src/awesomewm.d.tl/entity/Variable_Info.tl +++ b/src/awesomewm.d.tl/entity/Variable_Info.tl @@ -1,4 +1,5 @@ local List = require "pl.List" +local Type_Info = require "entity.Type_Info" local record Variable_Info metamethod __call: function(Variable_Info): Variable_Info @@ -6,7 +7,7 @@ local record Variable_Info Variable_Info: Variable_Info name: string - types: List + types: List constraints: List end diff --git a/src/awesomewm.d.tl/generator/snippets.tl b/src/awesomewm.d.tl/generator/snippets.tl index 510a945..96043ea 100644 --- a/src/awesomewm.d.tl/generator/snippets.tl +++ b/src/awesomewm.d.tl/generator/snippets.tl @@ -2,6 +2,7 @@ local Function_Info = require "entity.Function_Info" local List = require "pl.List" local stringx = require "pl.stringx" local template = require "pl.template" +local Type_Info = require "entity.Type_Info" local utils = require "utils" local Variable_Info = require "entity.Variable_Info" @@ -12,13 +13,13 @@ function snippets.indent(str: string, level: number): string return stringx.rstrip(stringx.indent(str, level, string.rep(" ", 3))) end -function snippets.render_typed_variable(name: string, types: List): string +function snippets.render_typed_variable(name: string, types: List): string local tmpl = [[$(name): $(types)]] local tmpl_args = { name = name, - types = types:concat(" | "), + types = types:map(function(t: Type_Info.Type_Info): string return t.name end):concat(" | "), } return utils.do_or_fail(template.substitute, tmpl, tmpl_args) @@ -70,10 +71,11 @@ function snippets.render_record_properties(items: List +local function parse_parameter_types(parameter_type: string): List if parameter_type == "" then - return List({ "any" }) + local type_info: Type_Info.Type_Info = { name = "any" } + return List({ type_info }) end - return stringx.split(parameter_type, " or "):map(utils.sanitize_string) + return stringx.split(parameter_type, " or "):map( + function(type_name: string): Type_Info.Type_Info + return { name = utils.sanitize_string(type_name) } + end + ) end local function extract_item_name(item_name_node: scan.HTMLNode): string @@ -50,12 +56,12 @@ local function extract_function_parameters(function_parameters_node: scan.HTMLNo local name = extract_node_text(nodes[query_selectors.name] as scan.HTMLNode) local types = parse_parameter_types(extract_node_text(nodes[query_selectors.types] as scan.HTMLNode)) - if types == List({ "table" }) then + if #types == 1 and types[1].name == "table" then local record_name = utils.capitalize(name) current_record_parameter = record_name return { name = name, - types = List({ record_name }), + types = List({ Type_Info(record_name) }), } end @@ -132,7 +138,8 @@ local function extract_section_variables(dl: string): { Variable_Info.Variable_I variable_info.name = extract_item_name(nodes[query_selectors.variable_name]) variable_info.types = parse_parameter_types(extract_node_text(nodes[query_selectors.variable_summary_type])) - if variable_info.types:contains("string") then + if #variable_info.types == 1 and variable_info.types[1].name == "string" then + log:debug("extract variable string with constraints, this is an enum") variable_info.constraints = List(extract_property_constraints(nodes[query_selectors.variable_property_constraint])):map( function(constraint: string): string return (constraint:gsub(""", "")) @@ -166,6 +173,12 @@ function module.get_doc_from_page(html: string, module_name: string): Module_Doc local module_doc = Module_Doc() module_doc.record_name = utils.capitalize((module_name:gsub(".*%.", ""))) + local self_type: Type_Info.Type_Info = { name = module_doc.record_name } + local self_parameter: Function_Info.Parameter = { + name = "self", + types = List({ self_type }), + } + for i = 1, #nodes:get("h2.section-header") do local h2 = nodes:get("h2.section-header")[i] local section_name = utils.sanitize_string(h2:inner_text()) @@ -181,10 +194,7 @@ function module.get_doc_from_page(html: string, module_name: string): Module_Doc log:warn("Not implemented: Deprecated object properties") elseif section_name == "Object methods" then module_doc.methods = List(extract_section_functions(dl_html)):map(function(method: Function_Info.Function_Info): Function_Info.Function_Info - method.parameters:insert(1, { - name = "self", - types = List({ module_doc.record_name }), - }) + method.parameters:insert(1, self_parameter) return method end) elseif section_name == "Signals" then From db49da6cfc9a7b301984780c5602c6ee1b2b9d93 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 6 Nov 2022 18:25:58 +0100 Subject: [PATCH 3/7] feat(scraper): extract record from parameters --- src/awesomewm.d.tl/scraper/module_doc.tl | 81 +++++++++++++++--------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index fc015b7..33d14ca 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -1,6 +1,7 @@ local Function_Info = require "entity.Function_Info" local List = require "pl.List" local logger = require "logger" +local Map = require "pl.Map" local Module_Doc = require "entity.Module_Doc" local scan = require "web_sanitize.query.scan_html" local scraper_utils = require "scraper.utils" @@ -32,47 +33,65 @@ local function extract_item_name(item_name_node: scan.HTMLNode): string return item_name_node and ((item_name_node.attr.name as string):gsub("^.*[%.:]", "")) end +local function extract_function_parameter_Parameters(tr_node: scan.HTMLNode): { Function_Info.Parameter } + local query_selectors = { + name = "span.parameter", + types = "span.types" + } + + return scraper_utils.scrape_tuples( + tr_node:outer_html(), + { query_selectors.name, query_selectors.types }, + function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Parameter + return { + name = extract_node_text(nodes[query_selectors.name] as scan.HTMLNode), + types = parse_parameter_types(extract_node_text(nodes[query_selectors.types] as scan.HTMLNode)), + } + end) +end + local function extract_function_parameters(function_parameters_node: scan.HTMLNode): { Function_Info.Parameter } - local current_record_parameter: string = nil + local current_record_parameter: Type_Info.Type_Info | nil = nil return scraper_utils.scrape( function_parameters_node:outer_html(), "tr", function(line_node: scan.HTMLNode): Function_Info.Parameter - if line_node.attr ~= nil and line_node.attr.class == "see_also_sublist" then - log:debug("subtable found, skipping") + local parameters = extract_function_parameter_Parameters(line_node) + if #parameters == 0 then + return nil + elseif #parameters ~= 1 then + log:error(logger.message_with_metadata("Expected 1 parameter by node", + { len = #parameters, line_node = line_node, parameters = parameters })) + error("Expected 1 parameter by node") + end + local name, types = parameters[1].name, parameters[1].types + + if line_node.attr ~= nil and line_node.attr.class == "see_also_sublist" and current_record_parameter then + local record_parameter = current_record_parameter as Type_Info.Type_Info + if not record_parameter.record_entries then + record_parameter.record_entries = Map() + end + + (record_parameter.record_entries as Map>):set(name, types) + return nil end - local query_selectors = { - name = "span.parameter", - types = "span.types" + if #types == 1 and types[1].name == "table" then + local record_name = utils.capitalize(name) + current_record_parameter = Type_Info(record_name) + return { + name = name, + types = List({ current_record_parameter }), + } + end + + return { + name = name, + types = types, } - - return scraper_utils.scrape_tuples( - line_node:outer_html(), - { query_selectors.name, query_selectors.types }, - function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Parameter - local name = extract_node_text(nodes[query_selectors.name] as scan.HTMLNode) - local types = parse_parameter_types(extract_node_text(nodes[query_selectors.types] as scan.HTMLNode)) - - if #types == 1 and types[1].name == "table" then - local record_name = utils.capitalize(name) - current_record_parameter = record_name - return { - name = name, - types = List({ Type_Info(record_name) }), - } - end - - return { - name = name, - types = types, - } - end - )[1] - end - ) + end) end local function extract_function_return_types(function_return_types_node: scan.HTMLNode): { string } From 05c098e0264d9fb0e62be863909d6f0b43517dd9 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 6 Nov 2022 18:30:16 +0100 Subject: [PATCH 4/7] build(types): fix pl Map:iter() definition --- types/pl.d.tl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/types/pl.d.tl b/types/pl.d.tl index b88ecbc..08cde60 100644 --- a/types/pl.d.tl +++ b/types/pl.d.tl @@ -71,7 +71,7 @@ local record pl keys: pl.List values: pl.List - iter: function(pl.Map): (function(): {K, V}) + iter: function(pl.Map): (function(): K, V) items: function(pl.Map): (function(): {K, V}) get: function(pl.Map, K): V From 453c9f2949a21dc0ac0e4c6365418c2652855475 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 6 Nov 2022 18:29:44 +0100 Subject: [PATCH 5/7] feat(generator): render function record parameters --- src/awesomewm.d.tl/entity/Type_Info.tl | 3 +- src/awesomewm.d.tl/entity/Variable_Info.tl | 10 +++-- src/awesomewm.d.tl/generator/snippets.tl | 51 +++++++++++++++++++++- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/src/awesomewm.d.tl/entity/Type_Info.tl b/src/awesomewm.d.tl/entity/Type_Info.tl index d7eb14d..1aa2b1e 100644 --- a/src/awesomewm.d.tl/entity/Type_Info.tl +++ b/src/awesomewm.d.tl/entity/Type_Info.tl @@ -1,3 +1,4 @@ +local List = require "pl.List" local Map = require "pl.Map" local record Type_Info @@ -9,7 +10,7 @@ local record Type_Info -- Map : name -> type -- We can't use Variable_Info here because it's a circular dependency. - record_entries: Map | nil + record_entries: Map> | nil end local __Type_Info: metatable = { diff --git a/src/awesomewm.d.tl/entity/Variable_Info.tl b/src/awesomewm.d.tl/entity/Variable_Info.tl index 48e4ce0..0469f2d 100644 --- a/src/awesomewm.d.tl/entity/Variable_Info.tl +++ b/src/awesomewm.d.tl/entity/Variable_Info.tl @@ -2,7 +2,7 @@ local List = require "pl.List" local Type_Info = require "entity.Type_Info" local record Variable_Info - metamethod __call: function(Variable_Info): Variable_Info + metamethod __call: function(self: Variable_Info, name: string | nil, types: List | nil): Variable_Info Variable_Info: Variable_Info @@ -13,10 +13,12 @@ local record Variable_Info end local __Variable_Info: metatable = { - __call = function(_self: Variable_Info): Variable_Info + __call = function(_self: Variable_Info, name: string | nil, types: List | nil): Variable_Info + name = name or "" + types = types or List() return { - name = "", - types = List(), + name = name, + types = types, } end, } diff --git a/src/awesomewm.d.tl/generator/snippets.tl b/src/awesomewm.d.tl/generator/snippets.tl index 96043ea..0f43d93 100644 --- a/src/awesomewm.d.tl/generator/snippets.tl +++ b/src/awesomewm.d.tl/generator/snippets.tl @@ -1,12 +1,24 @@ local Function_Info = require "entity.Function_Info" local List = require "pl.List" +local Map = require "pl.Map" local stringx = require "pl.stringx" local template = require "pl.template" local Type_Info = require "entity.Type_Info" local utils = require "utils" local Variable_Info = require "entity.Variable_Info" -local snippets = {} +local record Module + indent: function(str: string, level: number): string + render_typed_variable: function(name: string, types: List): string + render_anonymous_function_signature: function(item: Function_Info.Function_Info): string + render_record_functions: function(items: List): string + render_enum: function(name: string, values: List): string + render_record_properties: function(items: List): string + render_record: function(name: string, items: List): string +end + + +local snippets: Module = {} function snippets.indent(str: string, level: number): string level = level or 1 @@ -42,7 +54,26 @@ end function snippets.render_record_functions(items: List): string return items:map(function(item: Function_Info.Function_Info): string - return snippets.render_anonymous_function_signature(item) + return string.format( + "%s%s", + item.parameters:map(function(param: Function_Info.Parameter): string + if #param.types == 0 then + return "" + end + + return param.types:map(function(t: Type_Info.Type_Info): string + if t.record_entries and (t.record_entries as Map>):len() > 0 then + local properties: List = List() + for name, types in (t.record_entries as Map>):iter() do + properties:append(Variable_Info(name, types)) + end + + return snippets.render_record(t.name, properties) + end + return "" + end):concat("\n") + end):concat("\n"), + snippets.render_anonymous_function_signature(item)) end):concat("\n") end @@ -79,4 +110,20 @@ function snippets.render_record_properties(items: List): string + local tmpl = [[ +record $(name) +$(indent(body)) +end +]] + + local tmpl_args = { + name = name, + body = snippets.render_record_properties(items), + indent = snippets.indent, + } + + return utils.do_or_fail(template.substitute, tmpl, tmpl_args) +end + return snippets From fc9e26b9c95541fb41160580b2abfaf2bad43bc9 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 6 Nov 2022 19:09:39 +0100 Subject: [PATCH 6/7] fix(generator): clean up parameter record --- src/awesomewm.d.tl/generator/snippets.tl | 39 +++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/awesomewm.d.tl/generator/snippets.tl b/src/awesomewm.d.tl/generator/snippets.tl index 0f43d93..daeff11 100644 --- a/src/awesomewm.d.tl/generator/snippets.tl +++ b/src/awesomewm.d.tl/generator/snippets.tl @@ -15,6 +15,7 @@ local record Module render_enum: function(name: string, values: List): string render_record_properties: function(items: List): string render_record: function(name: string, items: List): string + render_records_from_Parameters: function(items: List): string end @@ -52,27 +53,31 @@ function snippets.render_anonymous_function_signature(item: Function_Info.Functi return utils.do_or_fail(template.substitute, tmpl, tmpl_args) end +function snippets.render_records_from_Parameters(items: List): string + return items:map(function(param: Function_Info.Parameter): string + if #param.types == 0 then + return "" + end + + return param.types:map(function(t: Type_Info.Type_Info): string + if t.record_entries and (t.record_entries as Map>):len() > 0 then + local properties: List = List() + for name, types in (t.record_entries as Map>):iter() do + properties:append(Variable_Info(name, types)) + end + + return snippets.render_record(t.name, properties) + end + return "" + end):filter(function(p: string): boolean return #p > 0 end):concat("\n") + end):filter(function(p: string): boolean return #p > 0 end):concat("\n") +end + function snippets.render_record_functions(items: List): string return items:map(function(item: Function_Info.Function_Info): string return string.format( "%s%s", - item.parameters:map(function(param: Function_Info.Parameter): string - if #param.types == 0 then - return "" - end - - return param.types:map(function(t: Type_Info.Type_Info): string - if t.record_entries and (t.record_entries as Map>):len() > 0 then - local properties: List = List() - for name, types in (t.record_entries as Map>):iter() do - properties:append(Variable_Info(name, types)) - end - - return snippets.render_record(t.name, properties) - end - return "" - end):concat("\n") - end):concat("\n"), + snippets.render_records_from_Parameters(item.parameters), snippets.render_anonymous_function_signature(item)) end):concat("\n") end From 0b71a76b34d02bb63ed022b2bc9f9c2b47660736 Mon Sep 17 00:00:00 2001 From: Aire-One Date: Sun, 6 Nov 2022 19:19:40 +0100 Subject: [PATCH 7/7] fix(generator): utils.is_empty --- src/awesomewm.d.tl/generator/snippets.tl | 10 ++++++++-- src/awesomewm.d.tl/utils.tl | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/awesomewm.d.tl/generator/snippets.tl b/src/awesomewm.d.tl/generator/snippets.tl index daeff11..7eb86df 100644 --- a/src/awesomewm.d.tl/generator/snippets.tl +++ b/src/awesomewm.d.tl/generator/snippets.tl @@ -69,8 +69,14 @@ function snippets.render_records_from_Parameters(items: List 0 end):concat("\n") - end):filter(function(p: string): boolean return #p > 0 end):concat("\n") + end):filter( + function (s: string): boolean + return not utils.is_empty(s) + end):concat("\n") + end):filter( + function (s: string): boolean + return not utils.is_empty(s) + end):concat("\n") end function snippets.render_record_functions(items: List): string diff --git a/src/awesomewm.d.tl/utils.tl b/src/awesomewm.d.tl/utils.tl index f4debbb..ff97a48 100644 --- a/src/awesomewm.d.tl/utils.tl +++ b/src/awesomewm.d.tl/utils.tl @@ -43,6 +43,10 @@ function utils.capitalize(s: string): string return (s:gsub("^%l", string.upper)) end +function utils.is_empty(s: string): boolean + return s == nil or s == "" +end + -- At some point, we should probably write a wrapper to make penlight's function work with pcalls. function utils.do_or_fail(func: function(...: any): (T | nil, string), ...: any): T local logger = require "logger"