commit 3056d48d5ad4ba3eaa2230bb35a6a5305af4efa8
Author: Aire-One <aireone@aireone.xyz>
Date:   Sun Jun 26 17:37:53 2022 +0200

    initial

diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000..cb33a3b
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,18 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.lua]
+indent_size = 3
+
+[justfile]
+indent_size = 2
+
+[*.json]
+indent_size = 2
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..450ff71
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+lua_modules
diff --git a/.luacheckrc b/.luacheckrc
new file mode 100644
index 0000000..91fb963
--- /dev/null
+++ b/.luacheckrc
@@ -0,0 +1,10 @@
+std = "lua54"
+cache = true
+
+files[".luacheckrc"].std = "+luacheckrc"
+
+include_files = {
+   "debug.lua",
+   "set_paths.lua",
+   "src/",
+}
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
new file mode 100644
index 0000000..ec06c1f
--- /dev/null
+++ b/.vscode/extensions.json
@@ -0,0 +1,13 @@
+{
+    "recommendations": [
+        "streetsidesoftware.code-spell-checker",
+        "tomblind.local-lua-debugger-vscode",
+        "johnnymorganz.stylua",
+        "skellock.just",
+        "sumneko.lua",
+        "editorconfig.editorconfig",
+        "bungcip.better-toml",
+        "tamasfe.even-better-toml",
+        "dwenegar.vscode-luacheck"
+    ]
+}
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000..cb54d96
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,14 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug",
+      "type": "lua-local",
+      "request": "launch",
+      "program": {
+        "command": "just"
+      },
+      "args": ["debug"]
+    }
+  ]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000..783318b
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,29 @@
+{
+  "editor.formatOnSave": true,
+  "editor.formatOnPaste": true,
+  "[markdown]": {
+    "editor.wordWrap": "on",
+    "editor.renderWhitespace": "all",
+    "editor.acceptSuggestionOnEnter": "off"
+  },
+  "cSpell.words": [
+    "ansicolors",
+    "awesomewm",
+    "getcontent",
+    "htmlparser",
+    "justfile",
+    "lldebugger",
+    "Luacheck",
+    "luacheckrc",
+    "lualogging",
+    "Luarocks",
+    "setopt",
+    "Stylua",
+    "wibox",
+    "writefunction"
+  ],
+  "files.associations": {
+    ".luacheckrc": "lua",
+    "*.rockspec": "lua"
+  }
+}
diff --git a/README.MD b/README.MD
new file mode 100644
index 0000000..2ca126b
--- /dev/null
+++ b/README.MD
@@ -0,0 +1,30 @@
+# AwesomeWM.d.tl
+
+A project to generate Teal types definitions for the Awesome WM based on the API documentation.
+
+# Dependencies
+
+Project dependencies :
+
+-   Lua5.4
+-   Luarocks
+-   Just (optional, you can check the commands in the `justfile` and manually run them)
+
+Note that additional dependencies are installed as rocks with Luarocks.
+
+Development dependencies:
+
+-   Luacheck
+-   Stylua
+
+# Run the project
+
+```sh
+# Install rocks
+just install
+
+# Run the main script
+just run
+```
+
+Custom settings for the project can be changed in the `src/awesomewm.d.tl/properties.lua` file.
diff --git a/debug.lua b/debug.lua
new file mode 100644
index 0000000..35a2a86
--- /dev/null
+++ b/debug.lua
@@ -0,0 +1,9 @@
+if os.getenv "LOCAL_LUA_DEBUGGER_VSCODE" ~= "1" then
+   print "You must run this script with the lua local debugger."
+end
+
+require("lldebugger").start()
+
+for _, test_file in pairs { ... } do
+   dofile(test_file)
+end
diff --git a/justfile b/justfile
new file mode 100644
index 0000000..bc66456
--- /dev/null
+++ b/justfile
@@ -0,0 +1,34 @@
+lua_version := "5.4"
+rocks_tree := "lua_modules"
+rockspec_file := "rockspecs/awesomewm.d.tl-dev-1.rockspec"
+
+lua_bin := `command -v lua5.4`
+lua := lua_bin + " -l set_paths"
+luarocks := `command -v luarocks`
+
+lua_targets := `find src/ -iname '*.lua' | xargs` + " " + `ls *.lua | xargs`
+
+install:
+  {{ luarocks }} \
+    --lua-version {{ lua_version }} \
+    install \
+    --tree {{ rocks_tree }} \
+    --only-deps \
+    {{ rockspec_file }}
+
+clean:
+  rm -rf {{ rocks_tree }}
+
+check-lua *FLAGS:
+  luacheck {{ lua_targets }} {{ FLAGS }}
+
+check-format:
+  stylua --check {{ lua_targets }}
+
+check: check-lua check-format
+
+run:
+  {{ lua }} src/awesomewm.d.tl/init.lua
+
+debug:
+  {{ lua }} debug.lua src/awesomewm.d.tl/init.lua
diff --git a/rockspecs/awesomewm.d.tl-dev-1.rockspec b/rockspecs/awesomewm.d.tl-dev-1.rockspec
new file mode 100644
index 0000000..654e6e8
--- /dev/null
+++ b/rockspecs/awesomewm.d.tl-dev-1.rockspec
@@ -0,0 +1,22 @@
+package = "awesomewm.d.tl"
+version = "dev-1"
+source = {
+   url = "*** please add URL for source tarball, zip or repository here ***",
+}
+description = {
+   homepage = "*** please enter a project homepage ***",
+   license = "*** please specify a license ***",
+}
+dependencies = {
+   "lua ~> 5.4",
+   "lualogging 1.6.0",
+   "inspect 3.1.3",
+   "ansicolors 1.0.2",
+   "lua-curl 0.3.13",
+   "htmlparser 0.3.9",
+   "web_sanitize 1.3.0",
+}
+build = {
+   type = "builtin",
+   modules = {},
+}
diff --git a/set_paths.lua b/set_paths.lua
new file mode 100644
index 0000000..54c6239
--- /dev/null
+++ b/set_paths.lua
@@ -0,0 +1,15 @@
+-- set_paths.lua
+local version = _VERSION:match "%d+%.%d+"
+
+local function lua_module_paths(module_base_path)
+   local paths = (module_base_path .. "/?.lua;")
+      .. (module_base_path .. "/?/init.lua;")
+
+   return paths
+end
+
+package.path = lua_module_paths("lua_modules/share/lua/" .. version)
+   .. lua_module_paths "src/awesomewm.d.tl"
+   .. package.path
+
+package.cpath = "lua_modules/lib/lua/" .. version .. "/?.so;" .. package.cpath
diff --git a/src/awesomewm.d.tl/crawler/init.lua b/src/awesomewm.d.tl/crawler/init.lua
new file mode 100644
index 0000000..5231eb6
--- /dev/null
+++ b/src/awesomewm.d.tl/crawler/init.lua
@@ -0,0 +1,31 @@
+local curl = require "cURL"
+
+local crawler = {}
+
+function crawler.request(url)
+   local queue = {}
+   local easy = curl.easy():setopt_url(url):setopt_writefunction(function(buf)
+      table.insert(queue, buf)
+   end)
+
+   local ok, err = easy:perform()
+   if not ok then
+      easy:close()
+      error { message = "curl.easy failed", err = err }
+   end
+
+   local code, body = easy:getinfo_response_code(), table.concat(queue)
+   easy:close()
+
+   if code ~= 200 then
+      error {
+         message = "curl response code is not 200",
+         code = code,
+         body = body,
+      }
+   end
+
+   return queue
+end
+
+return crawler
diff --git a/src/awesomewm.d.tl/init.lua b/src/awesomewm.d.tl/init.lua
new file mode 100644
index 0000000..722340f
--- /dev/null
+++ b/src/awesomewm.d.tl/init.lua
@@ -0,0 +1,19 @@
+local inspect = require "inspect"
+local log = require "logger"
+local properties = require "properties"
+local utils = require "utils"
+
+log:info(
+   inspect { message = "Start extraction", base_url = properties.base_url }
+)
+
+local index = utils.fetch(properties.base_url .. properties.index_uri)
+local modules = utils.get_modules_from_index(index, properties.ignored_modules)
+
+log:info(inspect { modules_found = #modules })
+
+local m = modules[1]
+log:info(inspect { try = m })
+local page = utils.fetch(properties.base_url .. "/" .. m.uri)
+local items = utils.get_items_from_page(page)
+log:info(inspect { items })
diff --git a/src/awesomewm.d.tl/logger/init.lua b/src/awesomewm.d.tl/logger/init.lua
new file mode 100644
index 0000000..76a4f92
--- /dev/null
+++ b/src/awesomewm.d.tl/logger/init.lua
@@ -0,0 +1,18 @@
+local ansicolors = require "ansicolors"
+local console = require "logging.console"
+local ll = require "logging"
+
+local log = console {
+   logLevel = ll.DEBUG,
+   destination = "stdout",
+   timestampPattern = "[%y-%m-%d %H:%M:%S]",
+   logPatterns = {
+      [ll.DEBUG] = ansicolors "%date%{cyan} %level %message %{reset}(%source)\n",
+      [ll.INFO] = ansicolors "%date %level %message\n",
+      [ll.WARN] = ansicolors "%date%{yellow} %level %message\n",
+      [ll.ERROR] = ansicolors "%date%{red bright} %level %message %{reset}(%source)\n",
+      [ll.FATAL] = ansicolors "%date%{magenta bright} %level %message %{reset}(%source)\n",
+   },
+}
+
+return log
diff --git a/src/awesomewm.d.tl/properties.lua b/src/awesomewm.d.tl/properties.lua
new file mode 100644
index 0000000..8ac42f6
--- /dev/null
+++ b/src/awesomewm.d.tl/properties.lua
@@ -0,0 +1,63 @@
+local properties = {}
+
+properties.base_url = "https://awesomewm.org/apidoc"
+
+properties.index_uri = "/index.html"
+
+--- Pages from the navigation menu to ignore.
+-- Sets to ignore documentations and sample file. I also added libraries with
+-- low quality API documentation, I'll probably work on them later, lets start
+-- with what works the best first.
+properties.ignored_modules = {
+   -- Sample files
+   "rc.lua",
+   "theme.lua",
+
+   -- Utility libraries
+   "gears.debug",
+   "gears.filesystem",
+   "gears.geometry",
+   "gears.math",
+   "gears.object",
+   "gears.protected_call",
+   "gears.sort",
+   "gears.string",
+   "gears.table",
+   "gears.wallpaper",
+
+   -- Theme related libraries
+   "beautiful",
+   "gears.color",
+   "gears.shape",
+
+   -- Classes
+   "awful.widget.common",
+   "gears.cache",
+   "gears.matrix",
+   "menubar.icon_theme",
+   "menubar.index_theme",
+   "signals",
+   "wibox.drawable",
+   "wibox.hierarchy",
+   "wibox.widget.base",
+   "xproperties",
+
+   -- Documentation
+   "Authors",
+   "Readme",
+   "Contributing",
+   "The Widget system",
+   "Creating new widget",
+   "Default configuration file documentation",
+   "Change Awesome appearance",
+   "My first Awesome",
+   "The AwesomeWM client layout system",
+   "Startup options",
+   "Building and Testing",
+   "Using Cairo and LGI",
+   "Tips for upgrading your configuration",
+   "NEWS",
+   "FAQ",
+}
+
+return properties
diff --git a/src/awesomewm.d.tl/scraper/init.lua b/src/awesomewm.d.tl/scraper/init.lua
new file mode 100644
index 0000000..fc94475
--- /dev/null
+++ b/src/awesomewm.d.tl/scraper/init.lua
@@ -0,0 +1,18 @@
+local scraper = {}
+
+function scraper.extract_nodes(document, selector, extractor)
+   local nodes = document(selector)
+   local extracts = {}
+
+   for _, node in ipairs(nodes) do
+      local data = extractor(node)
+
+      if data then
+         table.insert(extracts, data)
+      end
+   end
+
+   return extracts
+end
+
+return scraper
diff --git a/src/awesomewm.d.tl/utils.lua b/src/awesomewm.d.tl/utils.lua
new file mode 100644
index 0000000..f78843d
--- /dev/null
+++ b/src/awesomewm.d.tl/utils.lua
@@ -0,0 +1,86 @@
+local crawler = require "crawler"
+local htmlparser = require "htmlparser"
+local inspect = require "inspect"
+local log = require "logger"
+local scraper = require "scraper"
+local web_sanitize = require "web_sanitize"
+
+local utils = {}
+
+function utils.has_item(table, item)
+   for k, v in pairs(table) do
+      if v == item then
+         return k
+      end
+   end
+
+   return nil
+end
+
+function utils.sanitize_page_name(string)
+   return (web_sanitize.extract_text(string):gsub("^%s*(.-)%s*$", "%1"))
+end
+
+function utils.fetch(url)
+   local success, result = pcall(crawler.request, url)
+
+   if not success then
+      log:error(inspect { "fetch failed", status = success, error = result })
+      return
+   end
+
+   log:info(inspect { message = "successfully fetched resource", url = url })
+
+   return table.concat(result, "")
+end
+
+function utils.get_modules_from_index(html, ignored)
+   local document = htmlparser.parse(html)
+
+   local modules = scraper.extract_nodes(
+      document,
+      "#navigation ul > li a",
+      function(node)
+         if node.name ~= "a" then
+            return nil
+         end
+
+         local name = utils.sanitize_page_name(node:getcontent())
+
+         if utils.has_item(ignored, name) then
+            return nil
+         end
+
+         local module = {
+            name = name,
+            uri = node.attributes.href,
+         }
+
+         return module
+      end
+   )
+
+   return modules
+end
+
+function utils.get_items_from_page(html)
+   local document = htmlparser.parse(html, 9999)
+
+   local titles = scraper.extract_nodes(
+      document,
+      "h2.section-header",
+      function(node)
+         return {
+            name = node.name,
+         }
+      end
+   )
+
+   local items = scraper.extract_nodes(document, "dl.function", function(node)
+      return { name = node.name }
+   end)
+
+   return { titles, items }
+end
+
+return utils
diff --git a/stylua.toml b/stylua.toml
new file mode 100644
index 0000000..3c5eb9c
--- /dev/null
+++ b/stylua.toml
@@ -0,0 +1,6 @@
+column_width = 80
+line_endings = "Unix"
+indent_type = "Spaces"
+indent_width = 3
+quote_style = "AutoPreferDouble"
+no_call_parentheses = true