diff --git a/.busted b/.busted index ff4c29f..1d410d2 100644 --- a/.busted +++ b/.busted @@ -1,3 +1,7 @@ +if os.getenv "LOCAL_LUA_DEBUGGER_VSCODE" == "1" then + require("lldebugger").start() +end + require("tl").loader() local version = _VERSION:match "%d+%.%d+" @@ -10,11 +14,14 @@ local function lua_module_paths(module_base_path) end return { - default = { + _all = { lpath = lua_module_paths("lua_modules/share/lua/" .. version) .. lua_module_paths "types" .. lua_module_paths "./src/awesomewm.d.tl", cpath = "lua_modules/lib/lua/" .. version .. "/?.so;", loaders = { "teal" }, }, + coverage = { + coverage = true, + }, } diff --git a/.editorconfig b/.editorconfig index dc552b6..2ff1b1a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,13 +8,7 @@ charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true -[*.lua] -indent_size = 3 - -[*.{tl,d.tl}] -indent_size = 3 - -[.busted] +[*.{lua,tl,d.tl,busted,luacheckrc,luacov}] indent_size = 3 [justfile] diff --git a/.gitignore b/.gitignore index f6ca0ea..97ac9ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ lua_modules generated build +luacov diff --git a/.luacov b/.luacov new file mode 100644 index 0000000..381eb6d --- /dev/null +++ b/.luacov @@ -0,0 +1,46 @@ +-- -- Per https://github.com/lunarmodules/luacov/blob/c915f4b95e8df5dbfaf2b1f04f0b7da04506fc7a/src/luacov/reporter.lua#L102 +-- -- > The option includeuntestedfiles requires the lfs module (from luafilesystem) to be installed. +-- -- So I'm ok with including it here, since it's already an hard dependency. +-- local lfs = require "lfs" + +-- local function walk(dir, callback) +-- for file in lfs.dir(dir) do +-- if file ~= "." and file ~= ".." then +-- local path = dir .. "/" .. file +-- local attr = lfs.attributes(path) +-- if attr.mode == "directory" then +-- walk(path, callback) +-- else +-- callback(path) +-- end +-- end +-- end +-- end + +-- local teal_files = {} +-- for _, path in pairs { "spec", "src" } do +-- walk(path, function(file) +-- if file:match "%.tl$" then +-- table.insert(teal_files, file) +-- end +-- end) +-- end + +return { + statsfile = "luacov/luacov.stats.out", + reporter = "html", + reportfile = "luacov/luacov.report.html", + include = { + "spec", + "src", + }, + exclude = { + "lua_modules", + "spec/tlconfig.lua", + "src/awesome.d.tl/types", + "types", + }, + -- runreport = true, -- For some reason, it makes my computer crash + -- includeuntestedfiles = teal_files, + includeuntestedfiles = true, +} diff --git a/.vscode/launch.json b/.vscode/launch.json index cb54d96..dad13e8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,21 @@ "program": { "command": "just" }, - "args": ["debug"] + "args": ["run"], + "scriptFiles": ["${workspaceFolder}/**/*.tl"] + }, + { + "name": "Debug specs", + "type": "lua-local", + "request": "launch", + "program": { + "command": "just" + }, + "args": [ + "test", + "_spec" // Adapt to the spec to debug + ], + "scriptFiles": ["${workspaceFolder}/**/*.tl"] } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 80272e5..f071b81 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,10 +1,8 @@ { - "editor.formatOnSave": true, - "editor.formatOnPaste": true, "[markdown]": { - "editor.wordWrap": "on", + "editor.acceptSuggestionOnEnter": "off", "editor.renderWhitespace": "all", - "editor.acceptSuggestionOnEnter": "off" + "editor.wordWrap": "on" }, "cSpell.words": [ "aire-one", @@ -13,10 +11,12 @@ "ansicolors", "awesomewm", "buildx", + "concat", "dryrun", "getcontent", "gitea", "htmlparser", + "includeuntestedfiles", "isdir", "justfile", "libc", @@ -24,24 +24,37 @@ "lpath", "Luacheck", "luacheckrc", + "luacov", "luafilesystem", "lualogging", "Luarocks", "luasec", "luasocket", "mkdir", + "modname", + "reportfile", "rockspec", + "rstrip", + "runreport", "setopt", + "statsfile", + "stringx", "Stylua", + "sublist", "tablex", + "tbody", "tmpl", "wibox", "woodpeckerci", "writefunction" ], + "debug.allowBreakpointsEverywhere": true, + "editor.formatOnPaste": true, + "editor.formatOnSave": true, "files.associations": { + ".busted": "lua", ".luacheckrc": "lua", - "*.rockspec": "lua", - ".busted": "lua" + ".luacov": "lua", + "*.rockspec": "lua" } } diff --git a/justfile b/justfile index 370bcda..d35a481 100644 --- a/justfile +++ b/justfile @@ -47,13 +47,10 @@ validate: awesomerc.tl # `find . -type f -iname '*.d.tl' | xargs` -test: - busted +test PATTERN="_spec": + busted --pattern={{ PATTERN }} -# TODO : how to run a debugger on Teal code? -debug: - {{ lua }} debug.lua build/awesomewm.d.tl/init.lua -debug-gen: - cyan gen --output set_paths.lua --gen-compat "off" set_paths.tl - sed -i 's/"src\/awesomewm.d.tl"/"build\/awesomewm.d.tl"/g' set_paths.lua - stylua set_paths.lua +# Requires a patched version of luacov (https://github.com/lunarmodules/luacov/issues/98#issuecomment-1530491759) +coverage: + busted --coverage + luacov diff --git a/spec/example_spec.tl b/spec/example_spec.tl deleted file mode 100644 index 7448c7d..0000000 --- a/spec/example_spec.tl +++ /dev/null @@ -1,30 +0,0 @@ -local assert = require("luassert") - -describe("Busted unit testing framework", function() - describe("should be awesome", function() - it("should be easy to use", function() - assert.truthy "Yup." - end) - - it("should have lots of features", function() - -- deep check comparisons! - assert.same({ table = "great" }, { table = "great" }) - - -- or check by reference! - assert.not_equal({ table = "great" }, { table = "great" }) - - assert.truthy "this is a string" -- truthy: not false or nil - - assert.is_true(1 == 1) - - assert.falsy(nil) - assert.has_error(function() - error "Wat" - end, "Wat") - end) - - it("should provide some shortcuts to common functions", function() - assert.is_unique { { thing = 1 }, { thing = 2 }, { thing = 3 } } - end) - end) -end) diff --git a/spec/example_with_my_code_spec.tl b/spec/example_with_my_code_spec.tl deleted file mode 100644 index a510409..0000000 --- a/spec/example_with_my_code_spec.tl +++ /dev/null @@ -1,10 +0,0 @@ -local assert = require("luassert") -local utils = require("utils") - -describe("test", function() - it("has_item", function() - local t = {1, 2, 3} - assert.equal(utils.has_item(t, 1), 1) - assert.is_nil(utils.has_item(t, 4)) - end) -end) diff --git a/spec/printer/teal_type_definition_spec.tl b/spec/printer/teal_type_definition_spec.tl new file mode 100644 index 0000000..f6367ac --- /dev/null +++ b/spec/printer/teal_type_definition_spec.tl @@ -0,0 +1,174 @@ +local assert = require("luassert") +local type Node = require("types.Node") +local stringx = require("pl.stringx") +local teal_type_definition_printer = require("printer.teal_type_definition") + +local printer = teal_type_definition_printer.printer + +-- We need to remove the last newline inserted by Penlight's dedent +local function dedent(str: string): string + return (stringx.dedent(str):sub(1, -2)) +end + +local function gen(ast: Node, expected_code: string): function() + return function() + local generated = printer(ast) + assert.same(dedent(expected_code), generated) + end +end + +describe("Teal type definition Printer", function() + it("should print a simple module type definition", gen( + { + children = {}, + dependencies = {}, + name = "Empty", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Empty + end + + return Empty + ]])) + + it("should print an Enum exposed by the module", gen( + { + children = { + { + children = { + { + name = "widget::layout_changed", + token = "identifier", + }, + { + name = "widget::redraw_needed", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + }, + dependencies = {}, + name = "Signal_Module", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Signal_Module + enum Signal + "widget::layout_changed" + "widget::redraw_needed" + end + end + + return Signal_Module + ]])) + + it("should print a property exposed by the module", gen( + { + children = { + { + name = "text", + types = { "string", "nil" }, + token = "variable", + } + }, + dependencies = {}, + name = "Property_Module", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Property_Module + text: string | nil + end + + return Property_Module + ]])) + + it("should print a function exposed by the module", gen( + { + children = { + { + parameters = { + { + types = { "integer" }, + name = "pid", + token = "variable", + }, + { + types = { "integer" }, + name = "sig", + token = "variable", + }, + }, + return_types = { "boolean" }, + name = "kill", + token = "function", + }, + }, + dependencies = {}, + name = "Function_Module", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Function_Module + kill: function(pid: integer, sig: integer): boolean + end + + return Function_Module + ]])) + + it("should print nested Record exposed by the module", gen( + { + children = { + { + children = {}, + name = "Nested", + token = "record", + } + }, + dependencies = {}, + name = "Nested_Module", + token = "module", + }, + [[ + -- This file was auto-generated. + + local record Nested_Module + record Nested + end + end + + return Nested_Module + ]])) + + it("should print require statement for module's dependencies", gen( + { + children = {}, + dependencies = { + dep = "dep", + deeper = "path.dep.deeper", + }, + name = "Module", + token = "module", + }, + [[ + -- This file was auto-generated. + local type deeper = require("path.dep.deeper") + local type dep = require("dep") + + local record Module + end + + return Module + ]])) +end) diff --git a/spec/scraper/module_doc_spec.tl b/spec/scraper/module_doc_spec.tl new file mode 100644 index 0000000..d563fd4 --- /dev/null +++ b/spec/scraper/module_doc_spec.tl @@ -0,0 +1,823 @@ +local assert = require("luassert") +local type Node = require("types.Node") +local scraper = require("scraper.module_doc") + +local get_doc_from_page = scraper.get_doc_from_page + +local function test(html: string, module_path: string, expected_ast: Node, expected_other_nodes: { Node } | nil): function() + return function() + local ast , other_nodes = get_doc_from_page(html, module_path) + assert.same(expected_ast, ast) + assert.same(expected_other_nodes or {}, other_nodes) + end +end + +describe("Scrap documentation", function() + it("should return a valid AST for an empty module", test( + "", + "empty", + { + children = { + { + children = {}, + name = "Signal", + token = "enum", + } + }, + name = "Empty", + module_path = "empty", + dependencies = {}, + token = "module", + })) + + it("should produce Variable and `property::` Signal nodes", test( + [[ +

+ Object properties +

+
+
+ 馃敆 + value + number + 路 1 signal +
+
+

Constraints:

+ + + + + + + + + + +
+ + Default value + + : 0
+ + Negative allowed + + : true
+ +
+
+ ]], + "property_signal", + { + children = { + { + children = { + { + name = "property::value", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + { + name = "value", + types = { "number" }, + token = "variable", + } + }, + name = "Property_signal", + module_path = "property_signal", + dependencies = {}, + token = "module", + })) + + it("should produce Enum nodes when an Object Property type is a String with constraints", test( + [[ +

+ Object properties +

+
+
+ 馃敆 + horizontal_fit_policy + string + 路 1 signal +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Default value + : "auto"
+ Valid values: +
+ + "auto" + + + : Honor the resize variable and preserve the aspect + ratio. +
+ + "none" + + : Do not resize at all.
+ + "fit" + + : Resize to the widget width.
+ +
+
+ ]], + "property_enum", + { + children = { + { + children = { + { + name = "property::horizontal_fit_policy", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + { + children = { + { + name = "auto", + token = "identifier", + }, + { + name = "none", + token = "identifier", + }, + { + name = "fit", + token = "identifier", + }, + }, + name = "Horizontal_fit_policy", + token = "enum", + }, + { + name = "horizontal_fit_policy", + types = { "Horizontal_fit_policy" }, + token = "variable", + }, + }, + name = "Property_enum", + module_path = "property_enum", + dependencies = {}, + token = "module", + })) + + it("should produce a `string` typed Variable node when a String Property has no constraint", test( + [[ +

+ Object properties +

+
+
+ 馃敆 + markup + string + 路 1 signal +
+
+ string +
+
+ ]], + "property_string", + { + children = { + { + children = { + { + name = "property::markup", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + { + name = "markup", + types = { "string" }, + token = "variable", + } + }, + name = "Property_string", + module_path = "property_string", + dependencies = {}, + token = "module", + })) + + it("should provide a Function node with the `self` as the first positional parameter", test( + [[ +

Object methods

+
+
+ 馃敆 + :swap (tag2) + + +
+
+

Parameters:

+ + + + + + + + + + + + + + + +
NameType(s)Description
tag2 + tag + The second tag
+
+
+ ]], + "awful.tag", + { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + parameters = { + { + types = { "Tag" }, + name = "self", + token = "variable", + }, + { + types = { "tag" }, -- This needs to be fixed : tag -> Tag + name = "tag2", + token = "variable", + } + }, + return_types = {}, + name = "swap", + token = "function", + }, + }, + name = "Tag", + module_path = "awful.tag", + dependencies = {}, + token = "module", + })) + + it("should produce Signal nodes", test( + [[ +

Signals

+
+
+ 馃敆 + widget::layout_changed + 路 Inherited from wibox.widget.base +
+
+
+ 馃敆 + widget::redraw_needed + 路 Inherited from wibox.widget.base +
+
+
+ ]], + "signal", + { + children = { + { + children = { + { + name = "widget::layout_changed", + token = "identifier", + }, + { + name = "widget::redraw_needed", + token = "identifier", + }, + }, + name = "Signal", + token = "enum", + }, + }, + name = "Signal", + module_path = "signal", + dependencies = {}, + token = "module", + })) + + it("should produce Function nodes", test( + [[ +

+ Static module functions +

+
+
+ 馃敆 + awesome.kill + (pid, sig) + -> boolean + +
+
+ Send a signal to a process. +

Parameters:

+ + + + + + + + + + + + + + + + + + + + + +
NameType(s)Description
pid + integer + + Process identifier. 0 and negative values have special meaning. See + man 3 kill. +
sig + integer + + Signal number. See + awesome.unix_signal + for a list of signals. +
+

Returns:

+
    + boolean + true if the signal was successfully sent, else false +
+
+
+ ]], + "awesome", -- The module name must be the same as the module name in the doc + { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + parameters = { + { + types = { "integer" }, + name = "pid", + token = "variable", + }, + { + types = { "integer" }, + name = "sig", + token = "variable", + }, + }, + return_types = { "boolean" }, + name = "kill", + token = "function", + } + }, + name = "Awesome", + module_path = "awesome", + dependencies = {}, + token = "module", + })) + + it("should produce a Record node when a function parameter is a named-parameter-table", test( + [[ +

+ Static module functions +

+
+
+ 馃敆 + awful.screen.focused + {[args]} + -> nil or screen + +
+
+

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameType(s)DescriptionDefault value
argsOptional + table + Undefined
clientOptional + boolean + + Use the client screen instead of the mouse screen. + + false +
mouseOptional + boolean + Use the mouse screen + true +
+

Returns:

+
    + optional + screen + The focused screen object, or + nil + in case no screen is present currently. +
+
+
+ ]], + "awful.screen", + { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + children = { + { + types = { "boolean" }, + name = "client", + token = "variable", + }, + { + types = { "boolean" }, + name = "mouse", + token = "variable", + } + }, + name = "Focused_Args", + token = "record", + }, + { + parameters = { + { + types = { "Focused_Args" }, + name = "args", + token = "variable", + }, + }, + return_types = { "screen" }, + name = "focused", + token = "function", + }, + }, + name = "Screen", + module_path = "awful.screen", + dependencies = {}, + token = "module", + })) + + it("should go back to a table typed parameter when the record is empty", test( + [[ +

Static module functions

+
+
+ 馃敆 + gears.table.crush (target, source, raw) + -> table + + +
+
+

Parameters:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameType(s)DescriptionDefault value
targettable The target table. Values from source will be copied + into this table.Not applicable
sourcetable The source table. Its values will be copied into + target.Not applicable
rawOptionalbool If true, values will be assigned with rawset. + This will bypass metamethods on target.false
+

Returns:

+
    + table + The target table. +
+
+
+ ]], + "gears.table", + { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + parameters = { + { + types = { "table" }, + name = "target", + token = "variable", + }, + { + types = { "table" }, + name = "source", + token = "variable", + }, + { + types = { "bool" }, + name = "raw", + token = "variable", + } + }, + return_types = { "table" }, + name = "crush", + token = "function", + } + }, + name = "Table", + module_path = "gears.table", + dependencies = {}, + token = "module", + })) + + it("should go back to a table typed parameter when the record is empty and it's the last parameter", test( + [[ +

Object methods

+
+
+ 馃敆 + :tags + (tags_table) + -> table + 路 1 signal +
+
+

Parameters:

+ + + + + + + + + + + + + + + +
NameType(s)Description
tags_table + table + + A table with tags to set, or nil to get the current + tags. +
+

Returns:

+
    + table + A table with all tags. +
+
+
+ ]], + "awful.client", + { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + { + parameters = { + { + types = { "Client" }, + name = "self", + token = "variable", + }, + { + types = { "table" }, + name = "tags_table", + token = "variable", + }, + }, + return_types = { "table" }, + name = "tags", + token = "function", + }, + }, + name = "Client", + module_path = "awful.client", + dependencies = {}, + token = "module", + })) + + it("should return Function nodes with the `other_nodes` list when the function module name doesn't match the module name", test( + [[ +

+ Static module functions +

+
+
+ 馃敆 + client.instances + () + -> integer + +
+
+ Get the number of instances. +

Returns:

+
    + integer + The number of client objects alive. +
+ +
+
+ ]], + "awful.client", + { + children = { + { + children = {}, + name = "Signal", + token = "enum", + }, + }, + name = "Client", + module_path = "awful.client", + dependencies = {}, + token = "module", + }, + { + { + parameters = {}, + return_types = { "integer" }, + name = "client.instances", + token = "function", + } + })) +end) diff --git a/src/awesomewm.d.tl/ast.tl b/src/awesomewm.d.tl/ast.tl new file mode 100644 index 0000000..6f5dae8 --- /dev/null +++ b/src/awesomewm.d.tl/ast.tl @@ -0,0 +1,81 @@ +local type Node = require("types.Node") + +local basic_nodes : { Node.Token : function(name: string, ...: any): Node } = { + module = function(name: string, module_path: string): Node + return { + token = "module", + name = name, + module_path = module_path, + dependencies = {}, + children = {}, + } + end, + record = function(name: string): Node + return { + token = "record", + name = name, + children = {}, + } + end, + enum = function(name: string): Node + return { + token = "enum", + name = name, + children = {}, + } + end, + identifier = function(name: string): Node + return { + token = "identifier", + name = name, + } + end, + variable = function(name: string): Node + return { + token = "variable", + name = name, + types = {}, + } + end, + ["function"] = function(name: string): Node + return { + token = "function", + name = name, + parameters = {}, + return_types = {}, + } + end, + metamethod = function(name: string): Node + return { + token = "metamethod", + name = name, + parameters = {}, + return_types = {}, + } + end, +} + +local function create_node(token: Node.Token, name: string, ...: any): Node + local node = basic_nodes[token](name, ...) + return node +end + +local function iter_children(node: Node): function(): integer, Node + if node.children == nil then + return function(): integer, Node end + end + return ipairs(node.children) +end + +local function in_order_visitor(node: Node, visitor: function(Node)) + for _, child in iter_children(node) do + in_order_visitor(child, visitor) + end + visitor(node) +end + +return { + create_node = create_node, + iter_children = iter_children, + in_order_visitor = in_order_visitor, +} diff --git a/src/awesomewm.d.tl/dag.tl b/src/awesomewm.d.tl/dag.tl new file mode 100644 index 0000000..9953a87 --- /dev/null +++ b/src/awesomewm.d.tl/dag.tl @@ -0,0 +1,30 @@ +local type Dag = require("types.Dag") +local type Node = require("types.Node") +local utils = require("utils") + +local function new(): Dag + local dag : Dag = { + modules = {}, + global_nodes = {}, + } + return dag +end + +local function push_module(dag: Dag, module_path: string, root_node: Node) + dag.modules[module_path] = root_node +end + +local function push_global_nodes(dag: Dag, nodes: { Node }) + utils.spread(dag.global_nodes, nodes) +end + +local function iter_modules(dag: Dag): function(): string, Node + return pairs(dag.modules) +end + +return { + new = new, + push_module = push_module, + push_global_nodes = push_global_nodes, + iter_modules = iter_modules, +} diff --git a/src/awesomewm.d.tl/generator/init.tl b/src/awesomewm.d.tl/generator/init.tl deleted file mode 100644 index 586a64b..0000000 --- a/src/awesomewm.d.tl/generator/init.tl +++ /dev/null @@ -1,6 +0,0 @@ -return { - global_env_def = require "generator.global_env_def", - module_init_definition = require "generator.module_init_definition", - snippets = require "generator.snippets", - teal_type_definitions = require "generator.teal_type_definitions", -} diff --git a/src/awesomewm.d.tl/generator/teal_type_definitions.tl b/src/awesomewm.d.tl/generator/teal_type_definitions.tl deleted file mode 100644 index 0a1d31a..0000000 --- a/src/awesomewm.d.tl/generator/teal_type_definitions.tl +++ /dev/null @@ -1,56 +0,0 @@ -local Module_Doc = require "entity.Module_Doc" -local template = require "pl.template" -local utils = require "utils" -local snippets = require "generator.snippets" - --- The long therm goal is to have so many `snippets.render_*` functions that --- we can render the whole file with the smallest template possible. -local tmpl = [[ --- Auto generated file (Do not manually edit this file!) - -# if module.requires:len() ~= 0 then -$(snippets.render_requires(module.requires)) -# end -- /requires - -local record $(module.record_name) -# if #module.signals ~= 0 then -$(snippets.indent(snippets.render_enum("Signal", module.signals))) - -# end -- /signals -# if #module.methods ~= 0 then - -- Object methods -$(snippets.indent(snippets.render_record_functions(module.methods))) - -# end -- /methods -# if #module.properties ~= 0 then - -- Object properties -$(snippets.indent(snippets.render_record_properties(module.properties))) - -# end -- /properties -# if #module.constructors ~= 0 then - -- Constructors -$(snippets.indent(snippets.render_record_functions(module.constructors))) - -# end -- /constructors -# if #module.static_functions ~= 0 then - -- Static functions -$(snippets.indent(snippets.render_record_functions(module.static_functions))) -# end -- /static_functions -end - -return $(module.record_name) -]] - -local module = {} - -function module.generate_teal(data: Module_Doc.Module_Doc): string - local tmpl_args = { - ipairs = ipairs, - module = data, - snippets = snippets, - } - return utils.do_or_fail(template.substitute, tmpl, tmpl_args) - -end - -return module diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl index b508aab..fd99f4c 100644 --- a/src/awesomewm.d.tl/init.tl +++ b/src/awesomewm.d.tl/init.tl @@ -1,14 +1,22 @@ +if os.getenv("LOCAL_LUA_DEBUGGER_VSCODE") == "1" then + require("lldebugger").start() +end + +local ast = require "ast" local crawler = require "crawler" +local dag = require "dag" local filesystem = require "filesystem" -local generator = require "generator" +local printer = require "printer" local List = require "pl.List" local logger = require "logger" local Map = require "pl.Map" -local Module_Doc = require "entity.Module_Doc" local Module_Info = require "entity.Module_Info" +local module_dependencies = require "visitors.module_dependencies" +local type Node = require "types.Node" local property = require "property" local scraper = require "scraper" local stringx = require "pl.stringx" +local type_mapping = require "visitors.type_mapping" local utils = require "utils" local log = logger.log("main") @@ -38,7 +46,7 @@ local function module_lists( end -- The module's children list produced can contain duplicates. --- We ignore them for now because they are dismissed when building a Map for the generator. +-- We ignore them for now because they are dismissed when building a Map for the printer. local function modules_tree(modules: List): Map> local tree: Map> = Map() for module in modules:iter() do @@ -65,27 +73,37 @@ local function modules_tree(modules: List): Map ) - local html = crawler.fetch(url) - local module_doc = scraper.module_doc.get_doc_from_page(html, record_name) - module_doc:fixup() - module_doc:populate_requires() - filesystem.file_writer.write( - generator.teal_type_definitions.generate_teal(module_doc), - output_path - ) + local tree = modules_tree(module_infos) + for module, children in tree:iter() do + -- TODO : this map should be coupled with the all_module_infos list + local requires: Map = Map() + for child in children:iter() do + local name = child:gmatch(".*%.(.*)$")() + requires:set(name, child) + end + filesystem.file_writer.write( + printer.module_init_definition.generate_teal(requires), + property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl" + ) + end end +--- TODO : rewrite the module_info thingy local all_module_infos, module_infos, global_module_infos = module_lists( property.base_url .. property.index_uri, - List(property.capi_modules), - List(property.ignored_modules) + List(),-- List(property.capi_modules), + List(utils.spread( + property.ignored_modules, + { + -- Modules that broke the parser. + --- TODO : fix the parser + "awful.screenshot", + })) ) -local tree = modules_tree(module_infos) log:info( logger.message_with_metadata( @@ -94,49 +112,52 @@ log:info( total_module_count = #all_module_infos, module_count = #module_infos, global_module_count = #global_module_infos, - tree_items = tree:len(), } ) ) +-- Build the DAG +local module_dag = dag.new() for module in module_infos:iter() do - do_one_file( - property.base_url .. "/" .. module.uri, - module.name, - property.out_directory .. "/" .. module.name:gsub("%.", "/") .. ".d.tl" + local module_ast, other_nodes = scraper.module_doc.get_doc_from_page( + crawler.fetch(property.base_url .. "/" .. module.uri), + (module.name:gsub(".*%sand%s", "")) ) + + dag.push_module(module_dag, module_ast.module_path, module_ast) + dag.push_global_nodes(module_dag, other_nodes) end -local global_env_def: List = List() -for module in global_module_infos:iter() do - if module.name:gmatch(".*%sand%s.*") then - do_one_file( - property.base_url .. "/" .. module.uri, - module.name, - property.out_directory .. "/" .. module.name:gsub(".*%sand%s", ""):gsub("%.", "/") .. ".d.tl" - ) - end - - local html = crawler.fetch(property.base_url .. "/" .. module.uri) - local module_doc = scraper.module_doc.get_doc_from_page(html, (module.name:gsub("%sand%s.*", ""))) - module_doc:fixup() - module_doc.record_name = utils.lowercase(module_doc.record_name) - global_env_def:append(module_doc) +-- Run the visitors +for _,root in dag.iter_modules(module_dag) do + ast.in_order_visitor(root, function(node: Node) + type_mapping.visit(node) + end) + ast.in_order_visitor(root, function(node: Node) + module_dependencies.visit(node, root, module_dag) + end) end -filesystem.file_writer.write( - generator.global_env_def.generate_teal(global_env_def), - property.out_directory .. "/global_env.d.tl" -) -for module, children in tree:iter() do - -- TODO : this map should be coupled with the all_module_infos list - local requires: Map = Map() - for child in children:iter() do - local name = child:gmatch(".*%.(.*)$")() - requires:set(name, child) - end +-- Build the global module from dag.global_nodes +--- TODO : todo + +--- TODO : this is fun, but we need to do something with it +-- Write the DAG to a file +-- local inspect = require("inspect") +-- filesystem.file_writer.write( +-- inspect(module_dag, { newline = "\n", indent = " ", depth = 2 }), +-- "generated_dag.lua" +-- ) + +log:info("Preprocessing finished") + +-- Write modules types definitions to files +for module_path, root in dag.iter_modules(module_dag) do filesystem.file_writer.write( - generator.module_init_definition.generate_teal(requires), - property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl" + printer.teal_type_definition.printer(root), + property.out_directory .. "/" .. module_path:gsub("%.", "/") .. ".d.tl" ) end + +do_module_init_definition(module_infos) +log:info("Module init files generated") diff --git a/src/awesomewm.d.tl/generator/global_env_def.tl b/src/awesomewm.d.tl/printer/global_env_def.tl similarity index 96% rename from src/awesomewm.d.tl/generator/global_env_def.tl rename to src/awesomewm.d.tl/printer/global_env_def.tl index 47387d6..79b9250 100644 --- a/src/awesomewm.d.tl/generator/global_env_def.tl +++ b/src/awesomewm.d.tl/printer/global_env_def.tl @@ -2,7 +2,7 @@ local List = require "pl.List" local Module_Doc = require "entity.Module_Doc" local template = require "pl.template" local utils = require "utils" -local snippets = require "generator.snippets" +local snippets = require "printer.snippets" -- The long therm goal is to have so many `snippets.render_*` functions that -- we can render the whole file with the smallest template possible. diff --git a/src/awesomewm.d.tl/printer/init.tl b/src/awesomewm.d.tl/printer/init.tl new file mode 100644 index 0000000..1787676 --- /dev/null +++ b/src/awesomewm.d.tl/printer/init.tl @@ -0,0 +1,6 @@ +return { + global_env_def = require("printer.global_env_def"), + module_init_definition = require("printer.module_init_definition"), + snippets = require("printer.snippets"), + teal_type_definition = require("printer.teal_type_definition"), +} diff --git a/src/awesomewm.d.tl/generator/module_init_definition.tl b/src/awesomewm.d.tl/printer/module_init_definition.tl similarity index 93% rename from src/awesomewm.d.tl/generator/module_init_definition.tl rename to src/awesomewm.d.tl/printer/module_init_definition.tl index 20d8ed4..dca48f5 100644 --- a/src/awesomewm.d.tl/generator/module_init_definition.tl +++ b/src/awesomewm.d.tl/printer/module_init_definition.tl @@ -1,7 +1,7 @@ local Map = require "pl.Map" local template = require "pl.template" local utils = require "utils" -local snippets = require "generator.snippets" +local snippets = require "printer.snippets" -- The long therm goal is to have so many `snippets.render_*` functions that -- we can render the whole file with the smallest template possible. diff --git a/src/awesomewm.d.tl/generator/snippets.tl b/src/awesomewm.d.tl/printer/snippets.tl similarity index 100% rename from src/awesomewm.d.tl/generator/snippets.tl rename to src/awesomewm.d.tl/printer/snippets.tl diff --git a/src/awesomewm.d.tl/printer/teal_type_definition.tl b/src/awesomewm.d.tl/printer/teal_type_definition.tl new file mode 100644 index 0000000..fbfab82 --- /dev/null +++ b/src/awesomewm.d.tl/printer/teal_type_definition.tl @@ -0,0 +1,167 @@ +local ast = require("ast") +local logger = require("logger") +local type Node = require("types.Node") +local stringx = require("pl.stringx") +local utils = require("utils") + +local log = logger.log("scraper") + +local function render_types(types: { string }): string + if not types or #types == 0 then + return "" + end + return ": " .. table.concat(types, " | ") +end + +local function dedent(str: string): string + return stringx.dedent(str):sub(1, -2) +end + +local function render_code(code: string, indent_level: integer): string + if not code or code == "" then + return "" + end + + local generated = "" + for line in stringx.lines(dedent(code)) do + generated = generated .. stringx.rstrip(string.rep(" ", 3 * indent_level) .. line) .. "\n" + end + return generated +end + +local function render_require(dependencies: { string : string }): string + local generated = "" + for dependency, path in utils.pairsByKeys(dependencies) do + generated = generated .. string.format("local type %s = require(\"%s\")\n", dependency, path) + end + return generated +end + +local record Node_Printer_Function + on_node: function(node: Node, indent_level: integer): string, integer + before_node: nil | function(node: Node, indent_level: integer): string, integer + after_node: nil | function(node: Node, indent_level: integer): string, integer +end + +-- pre-declare functions to prevent forward reference errors +local print_teal: function(node: Node, indent_level: integer | nil): string +local print_children: function(node: Node): string + +local node_printer : { Node.Token : Node_Printer_Function } = { + ["module"] = { + before_node = function(node: Node, indent_level: integer): string, integer + return render_code( + string.format( + "-- This file was auto-generated.\n%s\nlocal record %s", + render_require(node.dependencies), -- last require statement will have a newline + node.name), + indent_level), indent_level + 1 + end, + on_node = function(node: Node, indent_level: integer): string, integer + return render_code(print_children(node), indent_level), indent_level + end, + after_node = function(node: Node, indent_level: integer): string, integer + return render_code("end", indent_level - 1) .. + "\n" .. + render_code( + string.format("return %s", node.name), + indent_level - 1 + ), indent_level - 1 + end, + }, + ["record"] = { + before_node = function(node: Node, indent_level: integer): string, integer + return render_code( + string.format( + "record %s", + node.name), + indent_level), indent_level + 1 + end, + on_node = function(node: Node, indent_level: integer): string, integer + return render_code(print_children(node), indent_level), indent_level + end, + after_node = function(_: Node, indent_level: integer): string, integer + return render_code("end", indent_level - 1), indent_level - 1 + end, + }, + ["enum"] = { + before_node = function(node: Node, indent_level: integer): string, integer + return render_code( + string.format( + "enum %s", + node.name), + indent_level), indent_level + 1 + end, + on_node = function(node: Node, indent_level: integer): string, integer + return render_code(print_children(node), indent_level), indent_level + end, + after_node = function(_: Node, indent_level: integer): string, integer + return render_code("end", indent_level - 1), indent_level - 1 + end, + }, + ["identifier"] = { + on_node = function(node: Node, indent_level: integer): string, integer + return render_code(string.format('"%s"', node.name), indent_level), indent_level + end, + }, + ["variable"] = { + on_node = function(node: Node, indent_level: integer): string, integer + return render_code( + string.format( + "%s%s", + node.name, + render_types(node.types)), + indent_level), indent_level + end, + }, + ["function"] = { + on_node = function(node: Node, indent_level: integer): string, integer + local args = {} + for _, parameter in ipairs(node.parameters) do + table.insert(args, print_teal(parameter):sub(1, -2)) -- need to remove the newline ending + end + return render_code( + string.format( + "%s: function(%s)%s", + node.name, + table.concat(args, ", "), + render_types(node.return_types)), + indent_level), indent_level + end, + }, + ["metamethod"] = { + on_node = function(): string, integer + log:warn("Metamethods are not supported yet") + end, + } +} + +function print_teal(node: Node, indent_level: integer | nil): string + indent_level = indent_level or 0 + local printer = node_printer[node.token] + local generated = "" + local full_generated = "" + if printer.before_node then + generated, indent_level = (printer.before_node as function(Node, integer): string, integer)(node, indent_level) + full_generated = generated + end + generated, indent_level = printer.on_node(node, indent_level) + full_generated = full_generated .. generated + if printer.after_node then + generated, indent_level = (printer.after_node as function(Node, integer): string, integer)(node, indent_level) + full_generated = full_generated .. generated + end + return full_generated +end + +function print_children(node: Node): string + local generated = "" + for _, child in ast.iter_children(node) do + generated = generated .. print_teal(child) + end + return generated +end + +return { + printer = print_teal, +} diff --git a/src/awesomewm.d.tl/scraper/module_doc.tl b/src/awesomewm.d.tl/scraper/module_doc.tl index 6c7bd66..0eb0f66 100644 --- a/src/awesomewm.d.tl/scraper/module_doc.tl +++ b/src/awesomewm.d.tl/scraper/module_doc.tl @@ -1,14 +1,10 @@ -local Function_Info = require "entity.Function_Info" -local List = require "pl.List" +local ast = require("ast") +local type Node = require("types.Node") local logger = require "logger" -local Map = require "pl.Map" -local Module_Doc = require "entity.Module_Doc" local scan = require "web_sanitize.query.scan_html" local scraper_utils = require "scraper.utils" local stringx = require "pl.stringx" -local Type_Info = require "entity.Type_Info" local utils = require "utils" -local Variable_Info = require "entity.Variable_Info" local log = logger.log("scraper") @@ -16,176 +12,184 @@ local function extract_node_text(node: scan.HTMLNode): string return utils.sanitize_string(node:inner_text()) end -local function parse_parameter_types(parameter_type: string): List +local function parse_parameter_types(parameter_type: string): { string } if parameter_type == "" then - local type_info: Type_Info.Type_Info = Type_Info("any") - return List({ type_info }) + return { "any" } end - return stringx.split(parameter_type, " or "):map( - function(type_name: string): Type_Info.Type_Info - return Type_Info(utils.sanitize_string(type_name)) + local types = {} + for t in stringx.split(parameter_type, " or "):iter() do + table.insert(types, t) + end + return types +end + +local function extract_item_name(item_name_node: scan.HTMLNode): string, string | nil + if not item_name_node then + return + end + local module_name_node = scraper_utils.find(item_name_node:outer_html(), "span.function_modname")[1] + local module_name = module_name_node and module_name_node:inner_text():gsub("[%.:]$", "") + local name = item_name_node:inner_text():gsub("^.*[%.:](.+)%s*[%(%{].*[%)%}]", "%1") + return utils.sanitize_string(name), module_name and utils.sanitize_string(module_name) or nil +end + +local function extract_function_parameters(table_html: string, function_name: string): { Node }, { Node } + local parameters_types : { Node } = {} + local is_populating_parameters_types: boolean = false + local current_field: Node = nil + + local parameters = scraper_utils.scrape(table_html, "tr", function(tr: scan.HTMLNode): Node + local tr_html = tr:outer_html() + local name_node = scraper_utils.find(tr_html, "span.parameter")[1] + local types_node = scraper_utils.find(tr_html, "span.types")[1] + if not name_node or not types_node then + return nil end - ) -end -local function extract_item_name(item_name_node: scan.HTMLNode): string - return item_name_node and ((item_name_node.attr.name as string):gsub("^.*[%.:]", "")) -end + local name = extract_node_text(name_node) + local types = parse_parameter_types(extract_node_text(types_node)) -local function extract_function_parameter_Parameters(tr_node: scan.HTMLNode): { Variable_Info.Variable_Info } - local query_selectors = { - name = "span.parameter", - types = "span.types" - } + -- Add a field to the current parameter type record + if tr.attr ~= nil and tr.attr.class == "see_also_sublist" and is_populating_parameters_types then + local field = ast.create_node("variable", name) + field.types = types + table.insert(parameters_types[#parameters_types].children, field) + return nil + end - return scraper_utils.scrape_tuples( - tr_node:outer_html(), - { query_selectors.name, query_selectors.types }, - function(nodes: { string : scan.HTMLNode | nil }): Variable_Info.Variable_Info - return Variable_Info( - extract_node_text(nodes[query_selectors.name] as scan.HTMLNode), - parse_parameter_types(extract_node_text(nodes[query_selectors.types] as scan.HTMLNode)) - ) - end) -end + -- Still here and we are populating the parameter type record ? + -- Then oops, we wrongly tried to convert a table to a record + if is_populating_parameters_types then + table.remove(parameters_types, #parameters_types) + is_populating_parameters_types = false + current_field.types = { "table" } + end -local function extract_function_parameters(function_parameters_node: scan.HTMLNode): { Variable_Info.Variable_Info } - local current_record_parameter: Type_Info.Type_Info | nil = nil + -- Otherwise, add a new parameter + local field = ast.create_node("variable", name) + field.types = types + current_field = field - return scraper_utils.scrape( - function_parameters_node:outer_html(), - "tr", - function(line_node: scan.HTMLNode): Variable_Info.Variable_Info - local parameters = extract_function_parameter_Parameters(line_node) - if #parameters == 0 then - return nil - elseif #parameters ~= 1 then - log:error(logger.message_with_metadata("Expected 1 parameter by node", - { len = #parameters, line_node = line_node, parameters = parameters })) - error("Expected 1 parameter by node") - end - local name, types = parameters[1].name, parameters[1].types + -- If the parameter is a table, then we try to convert it to a record + if #types == 1 and types[1] == "table" then + local record_name = string.format( + "%s_%s", + utils.capitalize(function_name), + utils.capitalize(name)) + table.insert(parameters_types, ast.create_node("record", record_name)) + is_populating_parameters_types = true + field.types = { record_name } + end - if line_node.attr ~= nil and line_node.attr.class == "see_also_sublist" and current_record_parameter then - local record_parameter = current_record_parameter as Type_Info.Type_Info - if not record_parameter.record_entries then - record_parameter.record_entries = Map() - end + return field + end) - (record_parameter.record_entries as Map>):set(name, types) - - return nil - end - - if #types == 1 and types[1].name == "table" then - local record_name = utils.capitalize(name) - current_record_parameter = Type_Info(record_name) - return Variable_Info( - name, - List({ current_record_parameter }) - ) - end - - return Variable_Info(name, types) - end) -end - -local function extract_function_return_types(function_return_types_node: scan.HTMLNode): List - if not function_return_types_node then - return {} + if is_populating_parameters_types and #parameters_types[#parameters_types].children == 0 then + table.remove(parameters_types, #parameters_types) + current_field.types = { "table" } end - local selector = "span.types .type" - local html = function_return_types_node:outer_html() + return parameters, parameters_types +end - return List(scraper_utils.scrape(html, selector, extract_node_text)):map( - function(type_name: string): Type_Info.Type_Info - return Type_Info(type_name) - end) +local function extract_function_return_types(ol_html: string): { string } + return scraper_utils.scrape(ol_html, "span.types .type", extract_node_text) end local function extract_property_constraints(property_constraint_node: scan.HTMLNode): { string } return scraper_utils.scrape( property_constraint_node:outer_html(), - "tr.see_also_sublist", + "tr.see_also_sublist i code", extract_node_text ) end -local function extract_section_functions(dl: string): { Function_Info.Function_Info } - local query_selectors = { - header = "dt", - name = "a", +local function extract_section_functions(dl: string, module_name: string | nil): { Node }, { Node} + local list_query_selectors : { string : string } = { + function_name = "dt strong", body = "dd", - parameters = "table", - return_types = "ol", } - return scraper_utils.scrape_tuples( + local functions : { Node } = {} + local other_functions : { Node } = {} + + for nodes in scraper_utils.iter_tuples( dl, - { query_selectors.header, query_selectors.body }, - function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Function_Info - if not nodes[query_selectors.header] or not nodes[query_selectors.body] then - log:warn( - logger.message_with_metadata( - "Missing header or body", - { nodes = nodes } - ) - ) - error("Missing header or body") - end - local header = nodes[query_selectors.header] as scan.HTMLNode - local body = nodes[query_selectors.body] as scan.HTMLNode - local body_elements = scraper_utils.extract_nodes( - body:outer_html(), - { query_selectors.parameters, query_selectors.return_types } - ) - return Function_Info( - scraper_utils.scrape( - header:outer_html(), - query_selectors.name, - extract_item_name - )[1], - #body_elements:get(query_selectors.parameters) ~= 0 and - List(extract_function_parameters(body_elements:get(query_selectors.parameters)[1])) or - (List() as List), - #body_elements:get(query_selectors.return_types) ~= 0 and - extract_function_return_types(body_elements:get(query_selectors.return_types)[1]) or - (List() as List) - ) + utils.values(list_query_selectors) + ) do + local function_name , function_module_name = extract_item_name(nodes[list_query_selectors.function_name]) + local function_node = ast.create_node( + "function", + function_name + ) + + local body_html = nodes[list_query_selectors.body]:outer_html() + + local parameter_node = scraper_utils.find(body_html, "table")[1] + local parameters, parameters_types: { Node }, { Node } = {}, {} + if parameter_node then + parameters, parameters_types = extract_function_parameters(parameter_node:outer_html(), function_name) end - ) + function_node.parameters = parameters + + local return_node = scraper_utils.find(body_html, "ol")[1] + function_node.return_types = return_node and + extract_function_return_types(return_node:outer_html()) or + {} + + if module_name and function_module_name and module_name ~= function_module_name then + function_node.name = function_module_name .. "." .. function_node.name + utils.spread(other_functions, parameters_types) + table.insert(other_functions, function_node) + else + utils.spread(functions, parameters_types) + table.insert(functions, function_node) + end + end + + return functions, other_functions end -local function extract_section_variables(dl: string): { Variable_Info.Variable_Info } - local query_selectors = { - variable_name = "dt a", +local function extract_section_variables(dl: string): { Node }, { string } + local query_selectors : { string : string } = { + variable_name = "dt strong", variable_summary_type = "dt span.summary_type", variable_property_constraint = "dd span.property_type", } - return scraper_utils.scrape_tuples( + local variables : { Node } = {} + local signals : { string } = {} + + for nodes in scraper_utils.iter_tuples( dl, - { query_selectors.variable_name, query_selectors.variable_summary_type, query_selectors.variable_property_constraint }, - function(nodes: { string : scan.HTMLNode | nil }): Variable_Info.Variable_Info - local variable_info = Variable_Info() + utils.values(query_selectors) + ) do + local node = ast.create_node("variable", (extract_item_name(nodes[query_selectors.variable_name]))) + node.types = parse_parameter_types(extract_node_text(nodes[query_selectors.variable_summary_type])) - variable_info.name = extract_item_name(nodes[query_selectors.variable_name]) - variable_info.types = parse_parameter_types(extract_node_text(nodes[query_selectors.variable_summary_type])) - - if #variable_info.types == 1 and variable_info.types[1].name == "string" then - log:debug("extract variable string with constraints, this is an enum") - variable_info.constraints = List(extract_property_constraints(nodes[query_selectors.variable_property_constraint])):map( - function(constraint: string): string - return (constraint:gsub(""", "")) - end + if #node.types == 1 and node.types[1] == "string" then + log:debug("extract variable string with constraints, this is an enum", { name = node.name }) + local type_enum = ast.create_node("enum", utils.capitalize(node.name)) + for _, constraint in ipairs(extract_property_constraints(nodes[query_selectors.variable_property_constraint])) do + table.insert( + type_enum.children, + ast.create_node("identifier", (constraint:gsub(""", ""))) ) end - - return variable_info + if #type_enum.children == 0 then + log:debug("Enum has no children, get back to variable", { name = node.name }) + else + table.insert(variables, type_enum) + node.types = { type_enum.name } + end end - ) + + table.insert(variables, node) + table.insert(signals, string.format("property::%s", node.name)) -- TODO : actually scrape the signals from the doc + end + + return variables, signals end local function extract_section_signal(dl: string): { string } @@ -202,58 +206,93 @@ local enum Section "Signals" end -local section_scrapers: { Section : function(html: string, module_doc: Module_Doc.Module_Doc) } = { - ["Constructors"] = function(html: string, module_doc: Module_Doc.Module_Doc) - module_doc.constructors = List(extract_section_functions(html)) - end, - ["Static module functions"] = function(html: string, module_doc: Module_Doc.Module_Doc) - module_doc.static_functions = List(extract_section_functions(html)) - end, - ["Object properties"] = function(html: string, module_doc: Module_Doc.Module_Doc) - module_doc.properties = List(extract_section_variables(html)) - end, - ["Object methods"] = function(html: string, module_doc: Module_Doc.Module_Doc) - local self_parameter = Variable_Info("self", List({ Type_Info(module_doc.record_name) })) - module_doc.methods = List(extract_section_functions(html)):map( - function(method: Function_Info.Function_Info): Function_Info.Function_Info - method.parameters:insert(1, self_parameter) - return method +-- returns +-- - Nodes that should be added to the module +-- - Nodes that should be added to the global scope +-- - Strings that should be added to the record Signals +local section_scrapers : { Section : function(html: string, record_name: string, module_name: string): { Node }, { Node }, { string } } = { + ["Constructors"] = function(html: string): { Node }, { Node }, { string } + local constructors = extract_section_functions(html) + for _, constructor in ipairs(constructors) do + if constructor.token == "function" then + constructor.name = "new" end - ) + end + return constructors, {}, {} end, - ["Signals"] = function(html: string, module_doc: Module_Doc.Module_Doc) - module_doc.signals = List(extract_section_signal(html)) + ["Static module functions"] = function(html: string, _: string, module_name: string): { Node }, { Node }, { string } + local static_functions, other_functions = extract_section_functions(html, module_name) + return static_functions, other_functions, {} + end, + ["Object properties"] = function(html: string): { Node }, { Node }, { string } + local properties, signals = extract_section_variables(html) + return properties, {}, signals + end, + ["Object methods"] = function(html: string, record_name: string): { Node }, { Node }, { string } + local methods = extract_section_functions(html) + for _, method in ipairs(methods) do + if method.token == "function" then + local self_parameter = ast.create_node("variable", "self") + self_parameter.types = { record_name } + table.insert(method.parameters, 1, self_parameter) + end + end + return methods, {}, {} + end, + ["Signals"] = function(html: string): { Node }, { Node }, { string } + local signals = extract_section_signal(html) + return {}, {}, signals end, } +-- local function extract_node_module_name(node: Node): string +-- return (node.name:gsub("(.*)[%.:].+$", "%1")) +-- end + local module = {} -function module.get_doc_from_page(html: string, module_name: string): Module_Doc.Module_Doc - local nodes = scraper_utils.extract_nodes(html, { +function module.get_doc_from_page(html: string, module_path: string): Node, { Node } + local html_nodes = scraper_utils.extract_nodes(html, { "h2.section-header", "dl.function", }) - if #nodes:get "h2.section-header" ~= #nodes:get "dl.function" then + if #html_nodes:get "h2.section-header" ~= #html_nodes:get "dl.function" then error "The list aren't the same size!" end - local module_doc = Module_Doc() - module_doc.record_name = utils.capitalize((module_name:gsub(".*%.", ""))) + local record_name = utils.capitalize((module_path:gsub(".*%.", ""))) + local module_root = ast.create_node("module", record_name, module_path) + local other_nodes : { Node } = {} - for i = 1, #nodes:get("h2.section-header") do - local h2 = nodes:get("h2.section-header")[i] + local module_signals_node = ast.create_node("enum", "Signal") + table.insert(module_root.children, module_signals_node) + + for i = 1, #html_nodes:get("h2.section-header") do + local h2 = html_nodes:get("h2.section-header")[i] local section_name = utils.sanitize_string(h2:inner_text()) as Section -- promote to Section, we then test if the section_name is in the table - local dl_html = nodes:get("dl.function")[i]:outer_html() + local dl_html = html_nodes:get("dl.function")[i]:outer_html() if section_scrapers[section_name] then - section_scrapers[section_name](dl_html, module_doc) + local module_nodes, global_nodes, signals_name = section_scrapers[section_name](dl_html, record_name, module_path) + for _, node in ipairs(module_nodes) do + table.insert(module_root.children, node) + end + for _, node in ipairs(global_nodes) do + table.insert(other_nodes, node) + end + for _, signal_name in ipairs(signals_name) do + table.insert( + module_signals_node.children, + ast.create_node("identifier", signal_name) + ) + end else log:warn("Section scraper not implemented: " .. section_name) end end - return module_doc + return module_root, other_nodes end return module diff --git a/src/awesomewm.d.tl/scraper/utils.tl b/src/awesomewm.d.tl/scraper/utils.tl index 26ff900..271ed0d 100644 --- a/src/awesomewm.d.tl/scraper/utils.tl +++ b/src/awesomewm.d.tl/scraper/utils.tl @@ -15,15 +15,14 @@ function scraper_utils.scrape(html: string, query_selector: string, extract_c scanner.scan_html(html, function(stack: scan.NodeStack) if stack:is(query_selector) then local node = stack:current() - local success, info_or_error = pcall(extract_callback, node) + local success, ret_or_error = pcall(extract_callback, node) if not success then - local error_message = info_or_error as string - log:error(logger.message_with_metadata("Extraction error", { error = error_message })) - else - local info = info_or_error as T - table.insert(ret, info) + log:error(logger.message_with_metadata("Extraction error", { error = ret_or_error as string })) + return end + + table.insert(ret, ret_or_error as T) end end) @@ -48,28 +47,56 @@ function scraper_utils.extract_nodes(html: string, query_selectors: { string }): return siblings end -function scraper_utils.scrape_tuples(html: string, query_selectors: { string }, extract_callback: function(tuple: { string : scan.HTMLNode | nil }): T): { T } - local nodes = scraper_utils.extract_nodes(html, query_selectors) +function scraper_utils.find(html: string, query_selector: string): { scan.HTMLNode } + local nodes: { scan.HTMLNode } = {} - local ret: { T } = {} + scanner.scan_html(html, function(stack: scan.NodeStack) + if stack:is(query_selector) then + table.insert(nodes, stack:current()) + end + end) - for i = 1, #nodes:get(query_selectors[1]) do - local node_list: { string : scan.HTMLNode | nil } = {} - tablex.foreach(query_selectors, function(query_selector: string) - node_list[query_selector] = nodes:get(query_selector)[i] or nil - end) - local success, info_or_error = pcall(extract_callback, node_list) + return nodes +end - if not success then - local error_message = info_or_error as string - log:error(logger.message_with_metadata("Extraction error", { error = error_message })) - else - local info = info_or_error as T - table.insert(ret, info) +function scraper_utils.iter_tuples(html: string, query_selectors: { string }): function(): { string : scan.HTMLNode } + local siblings: { string : { scan.HTMLNode } } = {} + for _, query_selector in ipairs(query_selectors) do + siblings[query_selector] = {} + end + + scanner.scan_html( + html, + function(stack: scan.NodeStack) + for _, query_selector in ipairs(query_selectors) do + if stack:is(query_selector) then + table.insert(siblings[query_selector], stack:current()) + end + end + end + ) + + local siblings_count = #siblings[query_selectors[1]] + for _, query_selector in ipairs(query_selectors) do + if #siblings[query_selector] ~= siblings_count then + error("Query selectors do not have the same number of siblings") end end - return ret + local i = 0 + return function(): { string : scan.HTMLNode } + i = i + 1 + if i > siblings_count then + return nil + end + + local node_list: { string : scan.HTMLNode } = {} + for _, query_selector in ipairs(query_selectors) do + node_list[query_selector] = siblings[query_selector][i] + end + + return node_list + end end return scraper_utils diff --git a/src/awesomewm.d.tl/types/Dag.tl b/src/awesomewm.d.tl/types/Dag.tl new file mode 100644 index 0000000..0f52f13 --- /dev/null +++ b/src/awesomewm.d.tl/types/Dag.tl @@ -0,0 +1,8 @@ +local type Node = require("types.Node") + +local record Dag + modules: { string : Node } -- module_path (AKA "full name" `package.module.name`) -> root_node (token = "module") + global_nodes: { Node } +end + +return Dag diff --git a/src/awesomewm.d.tl/types/Node.tl b/src/awesomewm.d.tl/types/Node.tl new file mode 100644 index 0000000..fa841cc --- /dev/null +++ b/src/awesomewm.d.tl/types/Node.tl @@ -0,0 +1,29 @@ +local record Node + enum Token + "module" -- file root node, it is always a record and the generated .d.tl file returns it + "record" + "enum" + "identifier" -- blank token with only a name (used for enum values) + "variable" + "function" + "metamethod" + end + token: Token + name: string + + -- for "module", "record", "enum" + children: { Node } + + -- for "variable" + types: { string } + + -- for "function" and "metamethod" + parameters: { Node } + return_types: { string } + + -- for "module" + module_path: string + dependencies: { string : string } -- module_name -> module_path +end + +return Node diff --git a/src/awesomewm.d.tl/utils.tl b/src/awesomewm.d.tl/utils.tl index 828070e..d1e1019 100644 --- a/src/awesomewm.d.tl/utils.tl +++ b/src/awesomewm.d.tl/utils.tl @@ -35,6 +35,16 @@ function utils.map(list: { T }, iteratee: function(value: T, position: int return mapped end +function utils.values(t: table): { T } + local values: { T } = {} + + for _, v in pairs(t) do + table.insert(values, v as T) + end + + return values +end + function utils.sanitize_string(s: string): string return (stringx.strip(web_sanitize.extract_text(s))) end @@ -66,4 +76,29 @@ function utils.do_or_fail(func: function(...: any): (T | nil, string), ... return res as T -- promote to T since pcall succeeded at this point end +function utils.spread(t: { T }, ...: { T }): { T } + for _, a in ipairs({ ... }) do + for _, v in ipairs(a) do + table.insert(t, v) + end + end + return t +end + +function utils.pairsByKeys(list: { Key : Value }, comp: function(Key, Key): boolean): function(): Key, Value + local sortedKeys = {} + for n in pairs(list) do table.insert(sortedKeys, n) end + table.sort(sortedKeys, comp) + + local index = 0 + return function(): Key, Value + index = index + 1 + if sortedKeys[index] == nil then + return nil + end + + return sortedKeys[index], list[sortedKeys[index]] + end +end + return utils diff --git a/src/awesomewm.d.tl/visitors/module_dependencies.tl b/src/awesomewm.d.tl/visitors/module_dependencies.tl new file mode 100644 index 0000000..94cc701 --- /dev/null +++ b/src/awesomewm.d.tl/visitors/module_dependencies.tl @@ -0,0 +1,70 @@ +local type Dag = require("types.Dag") +local type Node = require("types.Node") +local utils = require("utils") + +local spread = utils.spread + +local function get_all_types_in_node(node: Node): { string } + local parameters_types = {} + if node.parameters then + for _, v in ipairs(node.parameters) do + spread(parameters_types, v.types) + end + end + return spread( + {}, + node.types or {}, + node.return_types or {}, + parameters_types or {}) +end + +local function replace_in_node_type(node: Node, old_type: string, new_type: string) + if node.parameters then + for _, v in ipairs(node.parameters) do + for i, t in ipairs(v.types) do + if t == old_type then + v.types[i] = new_type + end + end + end + end + + if node.types then + for i, t in ipairs(node.types) do + if t == old_type then + node.types[i] = new_type + end + end + end + + if node.return_types then + for i, t in ipairs(node.return_types) do + if t == old_type then + node.return_types[i] = new_type + end + end + end +end + +local record Module_Dependencies + visit: function(node: Node, mod: Node, d: Dag) +end + +function Module_Dependencies.visit(node: Node, mod: Node, d: Dag) + local all_types = get_all_types_in_node(node) + for _, type_name in ipairs(all_types) do + if type_name == mod.name then + goto continue + end + + local dependency = d.modules[type_name] or d.modules[utils.lowercase(type_name)] + if dependency then + mod.dependencies[dependency.name] = dependency.module_path + replace_in_node_type(node, dependency.module_path, dependency.name) + end + + ::continue:: + end +end + +return Module_Dependencies diff --git a/src/awesomewm.d.tl/visitors/type_mapping.tl b/src/awesomewm.d.tl/visitors/type_mapping.tl new file mode 100644 index 0000000..9c6e74a --- /dev/null +++ b/src/awesomewm.d.tl/visitors/type_mapping.tl @@ -0,0 +1,68 @@ +local type Node = require("types.Node") + +-- Special types I don't want to deal with for now +local gears_shape_function = "function(cr: any, width: integer, height: integer)" + +local type_map : { string : string } = { + awesome = "Awesome", + bool = "boolean", + client = "Client", + ["gears.shape"] = gears_shape_function, + ["gears.surface"] = "Surface", + image = "Image", + int = "integer", + screen = "Screen", + shape = gears_shape_function, + surface = "Surface", + tag = "Tag", + widget = "wibox.widget", + + -- fixes we shouldn't have to do (We need to PR Awesome to fix the doc) + timer = "gears.timer", +} + +local function get_type(t: string): string + return type_map[t] or t +end + +local function check_node(node: Node) + if not node.types then + return + end + + for i, t in ipairs(node.types) do + node.types[i] = get_type(t) + end +end + +local function check_function_parameters(node: Node) + if not node.parameters then + return + end + + for _, parameter in ipairs(node.parameters) do + check_node(parameter) + end +end + +local function check_function_returns(node: Node) + if not node.return_types then + return + end + + for i, ret in ipairs(node.return_types) do + node.return_types[i] = get_type(ret) + end +end + +local record Type_Mapping + visit: function(node: Node) +end + +function Type_Mapping.visit(node: Node) + check_node(node) + check_function_parameters(node) + check_function_returns(node) +end + +return Type_Mapping diff --git a/types/lldebugger.d.tl b/types/lldebugger.d.tl new file mode 100644 index 0000000..5f2a85a --- /dev/null +++ b/types/lldebugger.d.tl @@ -0,0 +1,5 @@ +local record Lldebugger + start: function() +end + +return Lldebugger