From fcad0b33f9eb7b65767d2967d295de1cd1a4388f Mon Sep 17 00:00:00 2001
From: Aire-One <aireone@aireone.xyz>
Date: Mon, 19 Dec 2022 22:46:03 +0100
Subject: [PATCH] run: make the program run on all the doc site

---
 src/awesomewm.d.tl/init.tl | 52 +++++++++++++-------------------------
 1 file changed, 17 insertions(+), 35 deletions(-)

diff --git a/src/awesomewm.d.tl/init.tl b/src/awesomewm.d.tl/init.tl
index 743a5ec..d87c41c 100644
--- a/src/awesomewm.d.tl/init.tl
+++ b/src/awesomewm.d.tl/init.tl
@@ -1,6 +1,8 @@
 local crawler = require "crawler"
 local filesystem = require "filesystem"
+local List = require "pl.List"
 local logger = require "logger"
+local Module_Info = require "entity.Module_Info"
 local property = require "property"
 local scraper = require "scraper"
 local generator = require "generator"
@@ -11,48 +13,28 @@ log:info(logger.message_with_metadata("Start", { property = property }))
 
 local index = crawler.fetch(property.base_url .. property.index_uri)
 
--- local modules =
---    scraper.get_modules_from_index(index, property.ignored_modules)
-local module_infos = scraper.module_info_list.get_modules_from_index(index)
+local ignored_modules = List(property.ignored_modules)
+local module_infos = List(scraper.module_info_list.get_modules_from_index(index)):filter(
+   function(module: Module_Info.Module_Info): boolean
+      return not ignored_modules:contains(module.name)
+   end
+)
 
 log:info("Finished Module List scrapping, found " .. #module_infos .. " modules")
--- for i = 1, 1 do -- #modules do
---    local m = modules[i]
---    log:info(inspect { try = m })
---    local page = crawler.fetch(property.base_url .. "/" .. m.uri)
---    local items = scraper.get_doc_from_page(page)
---    log:info(inspect { items })
--- end
 
-local function do_one_file(url: string, module_name: string, output: string)
+local function do_one_file(url: string, output_base_dir: string)
+   local module_name = url:gsub(".*/", ""):gsub(".html", "")
    local html = crawler.fetch(url)
    local module_doc = scraper.module_doc.get_doc_from_page(html, module_name)
    filesystem.file_writer.write(
       generator.teal_type_definitions.generate_teal(module_doc),
-      output
+      output_base_dir .. module_name:gsub("%.", "/") .. ".d.tl"
    )
 end
 
-do_one_file(
-   property.base_url .. "/widgets/wibox.widget.textbox.html",
-   "wibox.widget.textbox",
-   property.out_directory .. "/textbox.d.tl"
-)
-
-do_one_file(
-   property.base_url .. "/popups_and_bars/wibox.html",
-   "wibox",
-   property.out_directory .. "/wibox.d.tl"
-)
-
-do_one_file(
-   property.base_url .. "/widget_layouts/wibox.layout.fixed.html",
-   "wibox.layout.fixed",
-   property.out_directory .. "/fixed.d.tl"
-)
-
-do_one_file(
-   property.base_url .. "/core_components/client.html",
-   "client",
-   property.out_directory .. "/client.d.tl"
-)
+for i = 1, #module_infos do
+   do_one_file(
+      property.base_url .. "/" .. module_infos[i].uri,
+      property.out_directory .. "/"
+   )
+end