run: let's make the main entry point do something

src/awesomewm.d.tl/dag.tl

@@ -1,4 +1,6 @@
 local type Dag = require("types.Dag")
+local type Node = require("types.Node")
+local utils = require("utils")
 
 local function new(): Dag
    local dag <total>: Dag = {
@@ -8,6 +10,21 @@ local function new(): Dag
    return dag
 end
 
+local function push_module(dag: Dag, module_path: string, root_node: Node)
+   dag.modules[module_path] = root_node
+end
+
+local function push_global_nodes(dag: Dag, nodes: { Node })
+   utils.spread(dag.global_nodes, nodes)
+end
+
+local function iter_modules(dag: Dag): function(): string, Node
+   return pairs(dag.modules)
+end
+
 return {
    new = new,
+   push_module = push_module,
+   push_global_nodes = push_global_nodes,
+   iter_modules = iter_modules,
 }

src/awesomewm.d.tl/init.tl

@@ -4,12 +4,12 @@ end
 
 local ast = require "ast"
 local crawler = require "crawler"
+local dag = require "dag"
 local filesystem = require "filesystem"
 local printer = require "printer"
 local List = require "pl.List"
 local logger = require "logger"
 local Map = require "pl.Map"
-local Module_Doc = require "entity.Module_Doc"
 local Module_Info = require "entity.Module_Info"
 local module_dependencies = require "visitors.module_dependencies"
 local type Node = require "types.Node"
@@ -73,109 +73,91 @@ local function modules_tree(modules: List<Module_Info.Module_Info>): Map<string,
    return tree
 end
 
--- local function do_one_file(
---    url: string,
---    record_name: string,
---    output_path: string
--- )
---    local html = crawler.fetch(url)
---    local module_doc = scraper.module_doc.get_doc_from_page(html, record_name)
---    module_doc:fixup()
---    module_doc:populate_requires()
---    filesystem.file_writer.write(
---       printer.teal_type_definitions.generate_teal(module_doc),
---       output_path
---    )
--- end
+--- TODO : rewrite this to use the DAG
+local function do_module_init_definition(
+   module_infos: List<Module_Info.Module_Info>
+)
+   local tree = modules_tree(module_infos)
+   for module, children in tree:iter() do
+      -- TODO : this map should be coupled with the all_module_infos list
+      local requires: Map<string, string> = Map()
+      for child in children:iter() do
+         local name = child:gmatch(".*%.(.*)$")()
+         requires:set(name, child)
+      end
+      filesystem.file_writer.write(
+         printer.module_init_definition.generate_teal(requires),
+         property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl"
+      )
+   end
+end
 
--- local all_module_infos, module_infos, global_module_infos = module_lists(
---    property.base_url .. property.index_uri,
---    List(property.capi_modules),
---    List(property.ignored_modules)
--- )
--- local tree = modules_tree(module_infos)
-
--- log:info(
---    logger.message_with_metadata(
---       "Finished Module List scrapping",
---       {
---          total_module_count = #all_module_infos,
---          module_count = #module_infos,
---          global_module_count = #global_module_infos,
---          tree_items = tree:len(),
---       }
---    )
--- )
-
--- for module in module_infos:iter() do
---    do_one_file(
---       property.base_url .. "/" .. module.uri,
---       module.name,
---       property.out_directory .. "/" .. module.name:gsub("%.", "/") .. ".d.tl"
---    )
--- end
-
--- local global_env_def: List<Module_Doc.Module_Doc> = List()
--- for module in global_module_infos:iter() do
---    if module.name:gmatch(".*%sand%s.*") then
---       do_one_file(
---          property.base_url .. "/" .. module.uri,
---          module.name,
---          property.out_directory .. "/" .. module.name:gsub(".*%sand%s", ""):gsub("%.", "/") .. ".d.tl"
---       )
---    end
-
---    local html = crawler.fetch(property.base_url .. "/" .. module.uri)
---    local module_doc = scraper.module_doc.get_doc_from_page(html, (module.name:gsub("%sand%s.*", "")))
---    module_doc:fixup()
---    module_doc.record_name = utils.lowercase(module_doc.record_name)
---    global_env_def:append(module_doc)
--- end
--- filesystem.file_writer.write(
---    printer.global_env_def.generate_teal(global_env_def),
---    property.out_directory .. "/global_env.d.tl"
--- )
-
--- for module, children in tree:iter() do
---    -- TODO : this map should be coupled with the all_module_infos list
---    local requires: Map<string, string> = Map()
---    for child in children:iter() do
---       local name = child:gmatch(".*%.(.*)$")()
---       requires:set(name, child)
---    end
---    filesystem.file_writer.write(
---       printer.module_init_definition.generate_teal(requires),
---       property.out_directory .. "/" .. stringx.split(module, "."):slice(1, -1):join("/") .. "/init.d.tl"
---    )
--- end
-
-local module_ast, other_nodes = scraper.module_doc.get_doc_from_page(
-   crawler.fetch(property.base_url .. "/core_components/tag.html"),
-   "awful.tag"
+--- TODO : rewrite the module_info thingy
+local all_module_infos, module_infos, global_module_infos = module_lists(
+   property.base_url .. property.index_uri,
+   List(),-- List(property.capi_modules),
+   List(utils.spread(
+      property.ignored_modules,
+      {
+         -- Modules that broke the parser.
+         --- TODO : fix the parser
+         "awful.screenshot",
+      }))
 )
 
-ast.in_order_visitor(module_ast, function(node: Node)
-   type_mapping.visit(node)
-end)
-ast.in_order_visitor(module_ast, function(node: Node)
-   module_dependencies.visit(node, module_ast, {
-      modules = {
-         Screen = {
-            name = "Screen",
-            module_path = "awful.screen",
-            token = "module",
-         },
+log:info(
+   logger.message_with_metadata(
+      "Finished Module List scrapping",
+      {
+         total_module_count = #all_module_infos,
+         module_count = #module_infos,
+         global_module_count = #global_module_infos,
       }
-   })
-end)
-
-log:info(logger.message_with_metadata("Finished", {
-   module_ast = module_ast,
-   other_nodes = other_nodes,
-   deps = module_ast.dependencies,
-}))
-
-filesystem.file_writer.write(
-   printer.teal_type_definition.printer(module_ast),
-   property.out_directory .. "/" .. "generated_from_ast" .. ".d.tl"
+   )
 )
+
+-- Build the DAG
+local module_dag = dag.new()
+for module in module_infos:iter() do
+   local module_ast, other_nodes = scraper.module_doc.get_doc_from_page(
+      crawler.fetch(property.base_url .. "/" .. module.uri),
+      (module.name:gsub(".*%sand%s", ""))
+   )
+
+   dag.push_module(module_dag, module_ast.module_path, module_ast)
+   dag.push_global_nodes(module_dag, other_nodes)
+end
+
+-- Run the visitors
+for _,root in dag.iter_modules(module_dag) do
+   ast.in_order_visitor(root, function(node: Node)
+      type_mapping.visit(node)
+   end)
+   ast.in_order_visitor(root, function(node: Node)
+      module_dependencies.visit(node, root, module_dag)
+   end)
+end
+
+-- Build the global module from dag.global_nodes
+--- TODO : todo
+
+--- TODO : this is fun, but we need to do something with it
+-- Write the DAG to a file
+-- local inspect = require("inspect")
+-- filesystem.file_writer.write(
+--    inspect(module_dag, { newline = "\n", indent = "   ", depth = 2 }),
+--    "generated_dag.lua"
+-- )
+
+log:info("Preprocessing finished")
+
+-- Write modules types definitions to files
+for module_path, root in dag.iter_modules(module_dag) do
+   filesystem.file_writer.write(
+      printer.teal_type_definition.printer(root),
+      property.out_directory .. "/" .. module_path:gsub("%.", "/") .. ".d.tl"
+   )
+end
+
+do_module_init_definition(module_infos)
+log:info("Module init files generated")

src/awesomewm.d.tl/types/Dag.tl

@@ -1,7 +1,7 @@
 local type Node = require("types.Node")
 
 local record Dag
-   modules: { string : Node } -- module_name -> root_node (token = "module")
+   modules: { string : Node } -- module_path (AKA "full name" `package.module.name`) -> root_node (token = "module")
    global_nodes: { Node }
 end