.editorconfig

@@ -11,6 +11,9 @@ insert_final_newline = true
 [*.lua]
 indent_size = 3
 
+[*.{tl,d.tl}]
+indent_size = 3
+
 [justfile]
 indent_size = 2
 

.gitignore

@@ -1,2 +1,3 @@
 lua_modules
 generated
+build

.woodpecker/build.yml

@@ -0,0 +1,25 @@
+depends_on:
+  - lint
+
+pipeline:
+  prepare:
+    image: akorn/luarocks:lua5.4-alpine
+    commands:
+      - apk add just gcc libc-dev musl-dev curl-dev
+      - just install
+  build:
+    image: akorn/luarocks:lua5.4-alpine
+    commands:
+      - apk add just gcc musl-dev
+      - luarocks install cyan
+      - just build
+  # run:
+  #   image: akorn/luarocks:lua5.4-alpine
+  #   commands:
+  #     - apk add just
+  #     - just run
+  # verify:
+  #   image: alpine:3.16
+  #   commands:
+  #     - apk add tree
+  #     - tree generated

.woodpecker/lint.yml

@@ -1,23 +1,38 @@
 variables:
-  - &when_path
+  - &when_path_lua
     - ".woodpecker/lint.yml"
+    - "justfile"
     - "**/*.lua"
+  - &when_path_teal
+    - ".woodpecker/lint.yml"
+    - "justfile"
+    - "**/*.tl"
 
 pipeline:
-  lint:
+  lint-lua:
     image: ghcr.io/lunarmodules/luacheck:v0.26.1
     group: lint
     commands:
       - apk add just
       - just check-lua
     when:
-      path: *when_path
+      path: *when_path_lua
 
-  style:
+  style-lua:
     image: alpine:edge # Stylua is only available on edge repository for now
     group: lint
     commands:
       - apk add just stylua
       - just check-format
     when:
-      path: *when_path
+      path: *when_path_lua
+
+  lint-teal:
+    image: akorn/luarocks:lua5.4-alpine
+    group: lint
+    commands:
+      - apk add just gcc libc-dev musl-dev
+      - luarocks install cyan
+      - just check-teal
+    when:
+      path: *when_path_teal

justfile

@@ -1,11 +1,13 @@
 lua_version := "5.4"
-lua_bin := "lua" + lua_version
+lua := "lua" + " -l set_paths"
-lua := lua_bin + " -l set_paths"
+
+tl := "tl run" + " -l set_paths"
 
 rocks_tree := "lua_modules"
 rockspec_file := "rockspecs/awesomewm.d.tl-dev-1.rockspec"
 
-lua_targets := `find src/ -iname '*.lua' | xargs` + " " + `ls *.lua | xargs`
+lua_targets := `find src/ -type f -iname '*.lua' | xargs` + " " + `ls *.lua | xargs`
+teal_targets := `find src/ types/ -type f -iname '*.tl' | xargs`
 
 install:
   luarocks \
@@ -21,13 +23,27 @@ uninstall:
 check-lua *FLAGS:
   luacheck {{ lua_targets }} {{ FLAGS }}
 
+check-teal:
+  cyan check {{ teal_targets }}
+
 check-format:
   stylua --check {{ lua_targets }}
 
-check: check-lua check-format
+check: check-lua check-format check-teal
+
+build:
+  cyan build
+
+clean:
+  rm -rf build
 
 run:
-  {{ lua }} src/awesomewm.d.tl/init.lua
+  {{ tl }} src/awesomewm.d.tl/init.tl
 
+# TODO : how to run a debugger on Teal code?
 debug:
-  {{ lua }} debug.lua src/awesomewm.d.tl/init.lua
+  {{ lua }} debug.lua build/awesomewm.d.tl/init.lua
+debug-gen:
+  cyan gen --output set_paths.lua --gen-compat "off" set_paths.tl
+  sed -i 's/"src\/awesomewm.d.tl"/"build\/awesomewm.d.tl"/g' set_paths.lua
+  stylua set_paths.lua

set_paths.lua

@@ -1,4 +1,3 @@
--- set_paths.lua
 local version = _VERSION:match "%d+%.%d+"
 
 local function lua_module_paths(module_base_path)
@@ -9,7 +8,8 @@ local function lua_module_paths(module_base_path)
 end
 
 package.path = lua_module_paths("lua_modules/share/lua/" .. version)
-   .. lua_module_paths "src/awesomewm.d.tl"
+   .. lua_module_paths "types"
+   .. lua_module_paths "build/awesomewm.d.tl"
    .. package.path
 
 package.cpath = "lua_modules/lib/lua/" .. version .. "/?.so;" .. package.cpath

set_paths.tl

@@ -0,0 +1,16 @@
+-- set_paths.lua
+local version = _VERSION:match "%d+%.%d+"
+
+local function lua_module_paths(module_base_path: string): string
+   local paths = (module_base_path .. "/?.lua;")
+      .. (module_base_path .. "/?/init.lua;")
+
+   return paths
+end
+
+package.path = lua_module_paths("lua_modules/share/lua/" .. version)
+   .. lua_module_paths "types"
+   .. lua_module_paths "build/awesomewm.d.tl"
+   .. package.path
+
+package.cpath = "lua_modules/lib/lua/" .. version .. "/?.so;" .. package.cpath

src/awesomewm.d.tl/crawler/init.tl

@@ -6,7 +6,7 @@ local tablex = require "pl.tablex"
 
 local crawler = {}
 
-local function http_request(url)
+local function http_request(url: string): string
    local body, code_or_error = http.request(url)
 
    if not body then
@@ -14,7 +14,7 @@ local function http_request(url)
       error { "Request failed", err = err, url = url }
    end
 
-   local code = code_or_error
+   local code = code_or_error as integer
 
    if code < 200 and code >= 300 then
       error {
@@ -27,7 +27,7 @@ local function http_request(url)
    return body
 end
 
-local function get_resource_by_protocol(url)
+local function get_resource_by_protocol(url: string): boolean, string
    local protocol, resource = url:match "^(%a+)://(.*)$"
 
    if not not tablex.find({ "http", "https" }, protocol) then
@@ -45,7 +45,7 @@ local function get_resource_by_protocol(url)
    end
 end
 
-function crawler.fetch(url)
+function crawler.fetch(url: string): string
    local success, result = get_resource_by_protocol(url)
 
    if not success then

src/awesomewm.d.tl/entities/Function_Info.lua

@@ -1,23 +0,0 @@
-local class = require "pl.class"
-local List = require "pl.List"
-
-local Function_Info = class.Module_Doc()
-
-function Function_Info:_init()
-   self.name = ""
-   self.parameters = List()
-   self.return_types = List()
-end
-
-function Function_Info:append_return_type(return_type)
-   self.return_types:append(return_type)
-end
-
-function Function_Info:append_parameter(name, type)
-   self.parameters:append {
-      name = name,
-      type = type,
-   }
-end
-
-return Function_Info

src/awesomewm.d.tl/entities/Function_Info.tl

@@ -0,0 +1,38 @@
+local List = require "pl.List"
+
+local record Parameter
+   name: string
+   type: string
+end
+
+local record Function_Info
+   metamethod __call: function(Function_Info): Function_Info
+
+   Function_Info: Function_Info
+
+   name: string
+   parameters: List<Parameter>
+   return_types: List<string>
+
+   append_parameter: function(self: Function_Info, name: string, type: string)
+   append_return_type: function(self: Function_Info, return_type: string)
+end
+
+local __Function_Info: metatable<Function_Info> = {
+   __call = function(_self: Function_Info): Function_Info
+      return {}
+   end,
+}
+
+function Function_Info:append_parameter(name: string, type: string)
+   self.parameters:append {
+      name = name,
+      type = type,
+   }
+end
+
+function Function_Info:append_return_type(return_type: string)
+   self.return_types:append(return_type)
+end
+
+return setmetatable({} as Function_Info, __Function_Info)

src/awesomewm.d.tl/entities/Module_Doc.lua

@@ -1,12 +0,0 @@
-local class = require "pl.class"
-local List = require "pl.List"
-
-local Module_Doc = class.Module_Doc()
-
-function Module_Doc:_init()
-   self.constructors = List()
-   self.methods = List()
-   self.static_functions = List()
-end
-
-return Module_Doc

src/awesomewm.d.tl/entities/Module_Doc.tl

@@ -0,0 +1,21 @@
+local Function_Info = require "entities.Function_Info"
+local List = require "pl.List"
+
+local record Module_Doc
+   metamethod __call: function(Module_Doc): Module_Doc
+
+   Module_Doc: Module_Doc
+
+   constructors: List<Function_Info.Function_Info>
+   methods: List<Function_Info.Function_Info>
+   static_functions: List<Function_Info.Function_Info>
+end
+
+local __Module_Doc: metatable<Module_Doc> = {
+   __call = function(_: Module_Doc): Module_Doc
+      return {}
+   end,
+}
+
+return setmetatable({} as Module_Doc, __Module_Doc)
+

src/awesomewm.d.tl/entities/Module_Info.lua

@@ -1,10 +0,0 @@
-local class = require "pl.class"
-
-local Module_Info = class.Module_Info()
-
-function Module_Info:_init(name, uri)
-   self.name = name
-   self.uri = uri
-end
-
-return Module_Info

src/awesomewm.d.tl/entities/Module_Info.tl

@@ -0,0 +1,19 @@
+local record Module_Info
+   metamethod __call: function(self: Module_Info, name: string, uri: string): Module_Info
+
+   Module_Info: Module_Info
+
+   name: string
+   uri: string
+end
+
+local __Module_Info: metatable<Module_Info> = {
+   __call = function(_self: Module_Info, name: string, uri: string): Module_Info
+      return {
+         name = name,
+         uri = uri,
+      }
+   end,
+}
+
+return setmetatable({} as Module_Info, __Module_Info)

src/awesomewm.d.tl/generator/init.tl

@@ -5,7 +5,7 @@ local log = require "logger"
 local utils = require "utils"
 local snippets = require "generator.snippets"
 
-local tmpl = (function(mod)
+local tmpl = (function(mod: string): string
    local package_path = utils.do_or_fail(path.package_path, mod)
    local package_dir = path.dirname(package_path)
    return utils.do_or_fail(file.read, package_dir .. "/template.tl.tmpl", false)
@@ -13,10 +13,22 @@ end)(...)
 
 local generator = {}
 
-function generator.generate_teal(data)
+local record Generate_Teal_Data_Record
+   section: string
+   items: { snippets.Anonymous_Function_Record }
+end
+function generator.generate_teal(data: { Generate_Teal_Data_Record }): string
    -- TODO : add the required modules to the generated code
    -- TODO : replace this with a proper way to get the module name (will also probably need the module path)
-   local module_data = { name = "module_name" }
+   local record Module_Data_Record
+      name: string
+      static_functions: { snippets.Anonymous_Function_Record }
+      constructors: { snippets.Anonymous_Function_Record }
+      methods: { string }
+      properties: { snippets.Anonymous_Function_Record }
+      signals: { snippets.Anonymous_Function_Record }
+   end
+   local module_data: Module_Data_Record = { name = "module_name" }
    for _, item in ipairs(data) do
       if item.section == "Static functions" then
          -- TODO
@@ -49,19 +61,19 @@ function generator.generate_teal(data)
    return utils.do_or_fail(template.substitute, tmpl, env)
 end
 
-function generator.write(file_content, file_path)
+function generator.write(file_content:string, file_path: string)
    -- Make sure the directory we want to write the file to exists
    local directory = path.dirname(file_path)
    if not path.isdir(directory) then
       path.mkdir(directory)
    end
 
-   local success, error = file.write(file_path, file_content, false)
+   local success, error_message = file.write(file_path, file_content, false)
 
    if not success then
       log:error {
          "generator.write error",
-         error = error,
+         error = error_message,
       }
       return
    end

src/awesomewm.d.tl/generator/snippets.tl

@@ -2,10 +2,10 @@ local utils = require "utils"
 local template = require "pl.template"
 
 -- Refactor scraper code to use pl.List objects
-local function join(arr, delim)
+local function join<T>(arr: { string }, delim: string): string
    local ret = ""
-   for i, type in ipairs(arr) do
+   for i, t in ipairs(arr) do
-      ret = ret .. type
+      ret = ret .. t
       if i < #arr then
          ret = ret .. delim
       end
@@ -13,13 +13,27 @@ local function join(arr, delim)
    return ret
 end
 
-local snippets = {}
+local record Anonymous_Function_Parameter_Record
+   name: string
+   type: { string }
+end
 
-function snippets.types_list(types)
+local record Anonymous_Function_Record
+   name: string
+   parameters: { Anonymous_Function_Parameter_Record }
+   returns: { string }
+end
+
+local snippets = {
+   Anonymous_Function_Parameter_Record = Anonymous_Function_Parameter_Record,
+   Anonymous_Function_Record = Anonymous_Function_Record,
+}
+
+function snippets.types_list(types: { string }): string
    return join(types, ", ")
 end
 
-function snippets.anonymous_function(item)
+function snippets.anonymous_function(item: Anonymous_Function_Record): string
    local parameters_string = ""
    if item.parameters then
       for i, param in ipairs(item.parameters) do

src/awesomewm.d.tl/logger/init.lua

@@ -1,18 +0,0 @@
-local ansicolors = require "ansicolors"
-local console = require "logging.console"
-local ll = require "logging"
-
-local log = console {
-   logLevel = ll.DEBUG,
-   destination = "stdout",
-   timestampPattern = "[%y-%m-%d %H:%M:%S]",
-   logPatterns = {
-      [ll.DEBUG] = ansicolors "%date%{cyan} %level %message %{reset}(%source)\n",
-      [ll.INFO] = ansicolors "%date %level %message\n",
-      [ll.WARN] = ansicolors "%date%{yellow} %level %message\n",
-      [ll.ERROR] = ansicolors "%date%{red bright} %level %message %{reset}(%source)\n",
-      [ll.FATAL] = ansicolors "%date%{magenta bright} %level %message %{reset}(%source)\n",
-   },
-}
-
-return log

src/awesomewm.d.tl/logger/init.tl

@@ -0,0 +1,17 @@
+local ansicolors = require "ansicolors"
+local logging_console = require "logging.console"
+
+local log = logging_console {
+   logLevel = "DEBUG",
+   destination = "stdout",
+   timestampPattern = "[%y-%m-%d %H:%M:%S]",
+   logPatterns = {
+      DEBUG = ansicolors "%date%{cyan} %level %message %{reset}(%source)\n",
+      INFO = ansicolors "%date %level %message\n",
+      WARN = ansicolors "%date%{yellow} %level %message\n",
+      ERROR = ansicolors "%date%{red bright} %level %message %{reset}(%source)\n",
+      FATAL = ansicolors "%date%{magenta bright} %level %message %{reset}(%source)\n",
+   },
+}
+
+return log

src/awesomewm.d.tl/properties.lua

@@ -1,66 +0,0 @@
-local properties = {}
-
--- properties.base_url = "https://awesomewm.org/apidoc"
-properties.base_url = "file:///usr/share/doc/awesome/doc"
-
-properties.index_uri = "/index.html"
-
-properties.out_directory = "generated"
-
---- Pages from the navigation menu to ignore.
--- Sets to ignore documentations and sample file. I also added libraries with
--- low quality API documentation, I'll probably work on them later, lets start
--- with what works the best first.
-properties.ignored_modules = {
-   -- Sample files
-   "rc.lua",
-   "theme.lua",
-
-   -- Utility libraries
-   "gears.debug",
-   "gears.filesystem",
-   "gears.geometry",
-   "gears.math",
-   "gears.object",
-   "gears.protected_call",
-   "gears.sort",
-   "gears.string",
-   "gears.table",
-   "gears.wallpaper",
-
-   -- Theme related libraries
-   "beautiful",
-   "gears.color",
-   "gears.shape",
-
-   -- Classes
-   "awful.widget.common",
-   "gears.cache",
-   "gears.matrix",
-   "menubar.icon_theme",
-   "menubar.index_theme",
-   "signals",
-   "wibox.drawable",
-   "wibox.hierarchy",
-   "wibox.widget.base",
-   "xproperties",
-
-   -- Documentation
-   "Authors",
-   "Readme",
-   "Contributing",
-   "The Widget system",
-   "Creating new widget",
-   "Default configuration file documentation",
-   "Change Awesome appearance",
-   "My first Awesome",
-   "The AwesomeWM client layout system",
-   "Startup options",
-   "Building and Testing",
-   "Using Cairo and LGI",
-   "Tips for upgrading your configuration",
-   "NEWS",
-   "FAQ",
-}
-
-return properties

src/awesomewm.d.tl/properties.tl

@@ -0,0 +1,72 @@
+local record Properties
+   base_url: string
+   index_uri: string
+
+   out_directory: string
+
+   --- Pages from the navigation menu to ignore.
+   -- Sets to ignore documentations and sample file. I also added libraries with
+   -- low quality API documentation, I'll probably work on them later, lets start
+   -- with what works the best first.
+   ignored_modules: { string }
+end
+
+local properties: Properties = {
+   -- base_url = "https://awesomewm.org/apidoc",
+   base_url = "file:///usr/share/doc/awesome/doc",
+   index_uri = "/index.html",
+   out_directory = "generated",
+   ignored_modules = {
+      -- Sample files
+      "rc.lua",
+      "theme.lua",
+
+      -- Utility libraries
+      "gears.debug",
+      "gears.filesystem",
+      "gears.geometry",
+      "gears.math",
+      "gears.object",
+      "gears.protected_call",
+      "gears.sort",
+      "gears.string",
+      "gears.table",
+      "gears.wallpaper",
+
+      -- Theme related libraries
+      "beautiful",
+      "gears.color",
+      "gears.shape",
+
+      -- Classes
+      "awful.widget.common",
+      "gears.cache",
+      "gears.matrix",
+      "menubar.icon_theme",
+      "menubar.index_theme",
+      "signals",
+      "wibox.drawable",
+      "wibox.hierarchy",
+      "wibox.widget.base",
+      "xproperties",
+
+      -- Documentation
+      "Authors",
+      "Readme",
+      "Contributing",
+      "The Widget system",
+      "Creating new widget",
+      "Default configuration file documentation",
+      "Change Awesome appearance",
+      "My first Awesome",
+      "The AwesomeWM client layout system",
+      "Startup options",
+      "Building and Testing",
+      "Using Cairo and LGI",
+      "Tips for upgrading your configuration",
+      "NEWS",
+      "FAQ",
+   }
+}
+
+return properties

src/awesomewm.d.tl/scraper/module_doc.tl

@@ -1,13 +1,15 @@
 local Function_Info = require "entities.Function_Info"
+local List = require "pl.List"
 local Module_Doc = require "entities.Module_Doc"
+local scan = require "web_sanitize.query.scan_html"
 local scraper_utils = require "scraper.utils"
 local utils = require "utils"
 
-local function extract_function_name(function_name_node)
+local function extract_function_name(function_name_node: scan.HTMLNode): string
-   return function_name_node and (function_name_node.attr.name:gsub(".*:", ""))
+   return function_name_node and ((function_name_node.attr.name as string):gsub(".*:", ""))
 end
 
-local function extract_function_return_types(function_return_types_node)
+local function extract_function_return_types(function_return_types_node: scan.HTMLNode): { string }
    if not function_return_types_node then
       return {}
    end
@@ -15,12 +17,12 @@ local function extract_function_return_types(function_return_types_node)
    local selector = "span.types .type"
    local html = function_return_types_node:outer_html()
 
-   return scraper_utils.scrape(html, selector, function(node)
+   return scraper_utils.scrape(html, selector, function(node: scan.HTMLNode): string
       return utils.sanitize_string(node:inner_text())
    end)
 end
 
-local function extract_section_functions(dl)
+local function extract_section_functions(dl: string): { Function_Info.Function_Info }
    local query_selectors = {
       function_name = "dt a",
       function_return_type = "dd ol",
@@ -29,13 +31,15 @@ local function extract_section_functions(dl)
    return scraper_utils.scrape_tuples(
       dl,
       { query_selectors.function_name, query_selectors.function_return_type },
-      function(nodes)
+      function(nodes: { string : scan.HTMLNode | nil }): Function_Info.Function_Info
          local function_info = Function_Info()
 
          function_info.name =
             extract_function_name(nodes[query_selectors.function_name])
-         function_info.return_types = extract_function_return_types(
+         function_info.return_types = List(
-            nodes[query_selectors.function_return_type]
+            extract_function_return_types(
+               nodes[query_selectors.function_return_type]
+            )
          )
 
          return function_info
@@ -45,7 +49,7 @@ end
 
 local module = {}
 
-function module.get_doc_from_page(html)
+function module.get_doc_from_page(html: string): Module_Doc.Module_Doc
    local nodes = scraper_utils.extract_nodes(html, {
       "h2.section-header",
       "dl.function",
@@ -57,20 +61,21 @@ function module.get_doc_from_page(html)
 
    local module_doc = Module_Doc()
 
-   for i, h2 in ipairs(nodes:get "h2.section-header") do
+   for i = 1, #nodes:get("h2.section-header") do
+      local h2 = nodes:get("h2.section-header")[i]
       local section_name = utils.sanitize_string(h2:inner_text())
       local dl_html = nodes:get("dl.function")[i]:outer_html()
 
       if section_name == "Constructors" then
-         module_doc.constructors = extract_section_functions(dl_html)
+         module_doc.constructors = List(extract_section_functions(dl_html))
       elseif section_name == "Static module functions" then
-         module_doc.static_functions = extract_section_functions(dl_html)
+         module_doc.static_functions = List(extract_section_functions(dl_html))
       elseif section_name == "Object properties" then
-         print "Not implemented: Deprecated object properties"
+         print "Not implemented: Object properties"
       elseif section_name == "Deprecated object properties" then
          print "Not implemented: Deprecated object properties"
       elseif section_name == "Object methods" then
-         module_doc.methods = extract_section_functions(dl_html)
+         module_doc.methods = List(extract_section_functions(dl_html))
       elseif section_name == "Signals" then
          print "Not implemented: Signals"
       else

src/awesomewm.d.tl/scraper/module_info_list.tl

@@ -1,4 +1,5 @@
 local Module_Info = require "entities.Module_Info"
+local scan = require "web_sanitize.query.scan_html"
 local scraper_utils = require "scraper.utils"
 local utils = require "utils"
 
@@ -6,9 +7,9 @@ local module = {}
 
 local MODULE_A_TAG_QUERY_SELECTOR = "div#navigation ul li a"
 
-local function extract_module_info(node)
+local function extract_module_info(node: scan.HTMLNode): Module_Info.Module_Info
    local name = utils.sanitize_string(node:inner_text())
-   local uri = node.attr.href
+   local uri = node.attr.href as string
 
    if not (name and uri) then
       error("Can't extract module info from node: " .. node:outer_html())
@@ -17,7 +18,7 @@ local function extract_module_info(node)
    return Module_Info(name, uri)
 end
 
-function module.get_modules_from_index(html)
+function module.get_modules_from_index(html: string): { Module_Info.Module_Info }
    return scraper_utils.scrape(
       html,
       MODULE_A_TAG_QUERY_SELECTOR,

src/awesomewm.d.tl/scraper/utils.lua

@@ -1,68 +0,0 @@
-local List = require "pl.List"
-local log = require "logger"
-local Map = require "pl.Map"
-local scanner = require "web_sanitize.query.scan_html"
-local tablex = require "pl.tablex"
-
-local scraper_utils = {}
-
-function scraper_utils.scrape(html, query_selector, extract_callback)
-   local ret = {}
-
-   scanner.scan_html(html, function(stack)
-      if stack:is(query_selector) then
-         local node = stack:current()
-         local success, info = pcall(extract_callback, node)
-
-         if not success then
-            log:error { message = info }
-         else
-            table.insert(ret, info)
-         end
-      end
-   end)
-
-   return ret
-end
-
-function scraper_utils.extract_nodes(html, query_selectors)
-   local siblings = Map()
-
-   tablex.foreach(query_selectors, function(query_selector)
-      siblings:set(query_selector, List())
-   end)
-
-   scanner.scan_html(html, function(stack)
-      tablex.foreach(query_selectors, function(query_selector)
-         if stack:is(query_selector) then
-            siblings:get(query_selector):append(stack:current())
-         end
-      end)
-   end)
-
-   return siblings
-end
-
-function scraper_utils.scrape_tuples(html, query_selectors, extract_callback)
-   local nodes = scraper_utils.extract_nodes(html, query_selectors)
-
-   local ret = {}
-
-   for i = 1, #nodes:get(query_selectors[1]) do
-      local node_list = {}
-      tablex.foreach(query_selectors, function(query_selector)
-         node_list[query_selector] = nodes:get(query_selector)[i] or nil
-      end)
-      local success, info = pcall(extract_callback, node_list)
-
-      if not success then
-         log:error { message = info }
-      else
-         table.insert(ret, info)
-      end
-   end
-
-   return ret
-end
-
-return scraper_utils

src/awesomewm.d.tl/scraper/utils.tl

@@ -0,0 +1,73 @@
+local List = require "pl.List"
+local log = require "logger"
+local Map = require "pl.Map"
+local scan = require "web_sanitize.query.scan_html"
+local scanner = require "web_sanitize.query.scan_html"
+local tablex = require "pl.tablex"
+
+local scraper_utils = {}
+
+function scraper_utils.scrape<T>(html: string, query_selector: string, extract_callback: function(node: scan.HTMLNode): T): { T }
+   local ret: { T } = {}
+
+   scanner.scan_html(html, function(stack: scan.NodeStack)
+      if stack:is(query_selector) then
+         local node = stack:current()
+         local success, info_or_error = pcall(extract_callback, node)
+
+         if not success then
+            local error_message = info_or_error as string
+            log:error { message = error_message }
+         else
+            local info = info_or_error as T
+            table.insert(ret, info)
+         end
+      end
+   end)
+
+   return ret
+end
+
+function scraper_utils.extract_nodes(html: string, query_selectors: { string }): Map<string, List<scan.HTMLNode>>
+   local siblings: Map<string, List<scan.HTMLNode>> = Map()
+
+   tablex.foreach(query_selectors, function(query_selector: string)
+      siblings:set(query_selector, List())
+   end)
+
+   scanner.scan_html(html, function(stack: scan.NodeStack)
+      tablex.foreach(query_selectors, function(query_selector: string)
+         if stack:is(query_selector) then
+            siblings:get(query_selector):append(stack:current())
+         end
+      end)
+   end)
+
+   return siblings
+end
+
+function scraper_utils.scrape_tuples<T>(html: string, query_selectors: { string }, extract_callback: function(tuple: { string : scan.HTMLNode | nil }): T): { T }
+   local nodes = scraper_utils.extract_nodes(html, query_selectors)
+
+   local ret: { T } = {}
+
+   for i = 1, #nodes:get(query_selectors[1]) do
+      local node_list: { string : scan.HTMLNode | nil } = {}
+      tablex.foreach(query_selectors, function(query_selector: string)
+         node_list[query_selector] = nodes:get(query_selector)[i] or nil
+      end)
+      local success, info_or_error = pcall(extract_callback, node_list)
+
+      if not success then
+         local error_message = info_or_error as string
+         log:error { message = error_message }
+      else
+         local info = info_or_error as T
+         table.insert(ret, info)
+      end
+   end
+
+   return ret
+end
+
+return scraper_utils

src/awesomewm.d.tl/utils.lua

@@ -1,70 +0,0 @@
-local web_sanitize = require "web_sanitize"
-
-local utils = {}
-
-function utils.has_item(table, item)
-   for k, v in pairs(table) do
-      if v == item then
-         return k
-      end
-   end
-
-   return nil
-end
-
-function utils.filter(list, predicate)
-   local filtered = {}
-
-   for position, value in ipairs(list) do
-      if predicate(value, position) then
-         table.insert(filtered, value)
-      end
-   end
-
-   return filtered
-end
-
-function utils.map(list, iteratee)
-   local mapped = {}
-
-   for position, value in ipairs(list) do
-      table.insert(mapped, iteratee(value, position))
-   end
-
-   return mapped
-end
-
-function utils.sanitize_string(string)
-   return utils.trim(
-      utils.replace(web_sanitize.extract_text(string), "^%s*(.-)%s*$", "%1")
-   )
-end
-
--- Extracted from teh Penlight Lua library.
--- Sometime Lua string.gsub can't match unescaped strings.
--- https://stackoverflow.com/a/72666170
-function utils.escape(string)
-   return (string:gsub("[%-%.%+%[%]%(%)%$%^%%%?%*]", "%%%1"))
-end
-
-function utils.replace(string, old, new, n)
-   return (string:gsub(utils.escape(old), new:gsub("%%", "%%%%"), n))
-end
-
-function utils.trim(string)
-   return string:match "^%s*(.-)%s*$"
-end
-
-function utils.do_or_fail(func, ...)
-   local log = require "logger"
-   local res, err = func(...)
-
-   if not res then
-      log:error { "do_or_fail failed!", error = err }
-      error(err)
-   end
-
-   return res
-end
-
-return utils

src/awesomewm.d.tl/utils.tl

@@ -0,0 +1,71 @@
+local web_sanitize = require "web_sanitize"
+
+local utils = {}
+
+function utils.has_item(t: table, item: any): any
+   for k, v in pairs(t) do
+      if v == item then
+         return k
+      end
+   end
+
+   return nil
+end
+
+function utils.filter<T>(list: { T }, predicate: function(value: T, position: integer): boolean): { T }
+   local filtered: { T } = {}
+
+   for position, value in ipairs(list) do
+      if predicate(value, position) then
+         table.insert(filtered, value)
+      end
+   end
+
+   return filtered
+end
+
+function utils.map<T, U>(list: { T }, iteratee: function(value: T, position: integer): U): { U }
+   local mapped: { U } = {}
+
+   for position, value in ipairs(list) do
+      table.insert(mapped, iteratee(value, position))
+   end
+
+   return mapped
+end
+
+-- Extracted from teh Penlight Lua library.
+-- Sometime Lua string.gsub can't match unescaped strings.
+-- https://stackoverflow.com/a/72666170
+function utils.escape(s: string): string
+   return (s:gsub("[%-%.%+%[%]%(%)%$%^%%%?%*]", "%%%1"))
+end
+
+function utils.replace(s: string, old: string, new: string, n: number): string
+   return (s:gsub(utils.escape(old), new:gsub("%%", "%%%%"), n))
+end
+
+function utils.trim(s: string): string
+   return s:match "^%s*(.-)%s*$"
+end
+
+function utils.sanitize_string(s: string): string
+   return utils.trim(
+      utils.replace(web_sanitize.extract_text(s), "^%s*(.-)%s*$", "%1")
+   )
+end
+
+-- At some point, we should probably write a wrapper to make penlight's function work with pcalls.
+function utils.do_or_fail<T>(func: function<T>(...: any): (T | nil, string), ...: any): T
+   local log = require "logger"
+   local res, err = func(...)
+
+   if not res then
+      log:error { "do_or_fail failed!", error = err }
+      error(err)
+   end
+
+   return res
+end
+
+return utils

tlconfig.lua

@@ -0,0 +1,8 @@
+return {
+   build_dir = "build",
+   source_dir = "src",
+   include_dir = {
+      "src/awesomewm.d.tl",
+      "types",
+   },
+}

types/ansicolors.d.tl

@@ -0,0 +1,6 @@
+local record ansicolors
+   noReset: function(str: string): string
+   metamethod __call: function(_: ansicolors, str: string): string
+end
+
+return ansicolors

types/inspect.d.tl

@@ -0,0 +1,12 @@
+local record inspect
+    record InspectOptions
+        depth: number
+        newline: string
+        indent: string
+        process: function(item: any, path: {any}): any
+    end
+
+    metamethod __call: function(self: inspect, value: any, options: InspectOptions): string
+end
+
+return inspect

types/logging.d.tl

@@ -0,0 +1,97 @@
+local enum Level
+   "DEBUG"
+   "INFO"
+   "WARN"
+   "ERROR"
+   "FATAL"
+   "OFF"
+end
+
+local type Append = function<T>(params: T, ...: any): Log
+
+local record Log
+   append: Append
+   setLevel: function (self: Log, level: Level)
+   log: function(self: Log, level: Level, ...: any)
+   getPrint: function(self: Log, level: Level): function(...: any)
+
+   debug: function(self: Log, ...: any)
+   info: function(self: Log, ...: any)
+   warn: function(self: Log, ...: any)
+   error: function(self: Log, ...: any)
+   fatal: function(self: Log, ...: any)
+
+   DEBUG: Level
+   INFO: Level
+   WARN: Level
+   ERROR: Level
+   FATAL: Level
+   OFF: Level
+end
+
+local record logging
+   Level: Level
+   Append: Append
+   Log: Log
+
+   _COPYRIGHT: string
+   _DESCRIPTION: string
+   _VERSION: string
+
+   DEBUG: Level
+   INFO: Level
+   WARN: Level
+   ERROR: Level
+   FATAL: Level
+   OFF: Level
+
+   new: function(append: function(self: Log, level: Level, msg: string | function), startLevel: Level): Log
+   buildLogPatterns: function(patterns: table, default: string): table
+   defaultLogPatterns: function(patt: string | table): table
+   defaultTimestampPattern: function(patt: string): string
+   defaultLevel: function(level: Level): Level
+   defaultLogger: function(logger: Log): Log
+
+   -- Deprecated
+   getDeprecatedParams: function(lst: table, ...: any)
+
+   -- Appenders (dynamically added to logging when required)
+   record ConsoleParameters
+      enum ConsoleDestination
+         "stdout"
+         "stderr"
+      end
+
+      destination: ConsoleDestination
+      logPattern: string
+      logPatterns: { Level : string }
+      timestampPattern: string
+      logLevel: Level
+   end
+   console: Append<ConsoleParameters>
+
+   record FileParameters
+      filename: string
+      datePattern: string
+      logPattern: string
+      logPatterns: { Level : string }
+      timestampPattern: string
+      logLevel: Level
+   end
+   file: Append<FileParameters>
+
+   record RollingFileParameters
+      filename: string
+      maxFileSize: number
+      maxBackupIndex: number
+      logPattern: string
+      logPatterns: { Level : string }
+      timestampPattern: string
+      logLevel: Level
+   end
+   rolling_file: Append<RollingFileParameters>
+
+   -- TODO : add more appenders
+end
+
+return logging

types/logging/console.d.tl

@@ -0,0 +1 @@
+return require "logging".console

types/logging/file.d.tl

@@ -0,0 +1 @@
+return require "logging".file

types/logging/rolling_file.d.tl

@@ -0,0 +1 @@
+return require("logging").rolling_file

types/ltn12.d.tl

@@ -0,0 +1,45 @@
+
+local record ltn12
+   type Filter = function(string): string, string
+   type Sink = function(string, string): boolean, string
+   type Source = function(): string, string
+
+   type FancySink = function(string, string): boolean, string | FancySink
+   type FancySource = function(): string, string | FancySource
+
+   -- Docs just say returns a truthy value on success
+   -- Since this value should really only be
+   -- used to check for truthiness, any seems fine here
+   type Pump = function(Source, Sink): any, string
+
+   record filter
+      chain: function(Filter, Filter, ...: Filter): Filter
+
+      cycle: function(string, string, any): Filter
+   end
+   record pump
+      all: Pump
+      step: Pump
+   end
+   record sink
+      chain: function(Filter, Sink): Sink
+      error: function(string): Sink
+      file: function(FILE, string): Sink
+      null: function(): Sink
+      simplify: function(FancySink): Sink
+      table: function({string}): Sink, {string}
+   end
+   record source
+      cat: function(Source, ...: Source): Source
+      chain: function(Source, Filter): Source
+      empty: function(): Source
+      error: function(string): Source
+      file: function(FILE): Source
+      file: function(FILE, string): Source
+      simplify: function(FancySource): Source
+      string: function(string): Source
+      table: function({string}): Source, {string}
+   end
+end
+
+return ltn12

types/mime.d.tl

@@ -0,0 +1,42 @@
+local ltn12 = require("ltn12")
+local type Filter = ltn12.Filter
+
+local record mime
+   normalize: function(): Filter
+   normalize: function(string): Filter
+
+   enum Encoding
+      "base64"
+      "quoted-printable"
+   end
+   enum Mode
+      "text"
+      "binary"
+   end
+
+   decode: function(Encoding): Filter
+   encode: function(Encoding, Mode): Filter
+
+   stuff: function(): Filter
+
+   wrap: function(string, integer): Filter
+   wrap: function(Encoding): Filter
+
+   b64: function(string, string): string, string
+
+   dot: function(integer, string): string, integer
+
+   eol: function(integer, string, string): string, integer
+
+   qp: function(string, string, string): string, string
+
+   qpwrp: function(integer, string, integer): string, integer
+
+   unb64: function(string, string): string, string
+
+   unqp: function(string, string): string, string
+
+   wrp: function(integer, string, integer): string, integer
+end
+
+return mime

types/pl/Date.d.tl

@@ -0,0 +1 @@
+return require "pl".Date

types/pl/List.d.tl

@@ -0,0 +1 @@
+return require "pl".List

types/pl/Map.d.tl

@@ -0,0 +1 @@
+return require "pl".Map

types/pl/app.d.tl

@@ -0,0 +1 @@
+return require "pl".app

types/pl/array2d.d.tl

@@ -0,0 +1 @@
+return require "pl".array2d

types/pl/class.d.tl

@@ -0,0 +1 @@
+return require "pl".class

types/pl/compat.d.tl

@@ -0,0 +1 @@
+return require "pl".compat

types/pl/comprehension.d.tl

@@ -0,0 +1 @@
+return require "pl".comprehension

types/pl/config.d.tl

@@ -0,0 +1 @@
+return require "pl".config

types/pl/data.d.tl

@@ -0,0 +1 @@
+return require "pl".data

types/pl/dir.d.tl

@@ -0,0 +1 @@
+return require "pl".dir

types/pl/file.d.tl

@@ -0,0 +1 @@
+return require "pl".file

types/pl/func.d.tl

@@ -0,0 +1 @@
+return require "pl".func

types/pl/input.d.tl

@@ -0,0 +1 @@
+return require "pl".input

types/pl/lapp.d.tl

@@ -0,0 +1 @@
+return require "pl".lapp

types/pl/lexer.d.tl

@@ -0,0 +1 @@
+return require "pl".lexer

types/pl/luabalanced.d.tl

@@ -0,0 +1 @@
+return require "pl".luabalanced

types/pl/operator.d.tl

@@ -0,0 +1 @@
+return require "pl".operator

types/pl/path.d.tl

@@ -0,0 +1 @@
+return require "pl".path

types/pl/permute.d.tl

@@ -0,0 +1 @@
+return require "pl".permute

types/pl/pretty.d.tl

@@ -0,0 +1 @@
+return require "pl".pretty

types/pl/seq.d.tl

@@ -0,0 +1 @@
+return require "pl".seq

types/pl/sip.d.tl

@@ -0,0 +1 @@
+return require "pl".sip

types/pl/strict.d.tl

@@ -0,0 +1 @@
+return require "pl".strict

types/pl/stringio.d.tl

@@ -0,0 +1 @@
+return require "pl".stringio

types/pl/stringx.d.tl

@@ -0,0 +1 @@
+return require "pl".stringx

types/pl/tablex.d.tl

@@ -0,0 +1 @@
+return require("pl").tablex

types/pl/template.d.tl

@@ -0,0 +1 @@
+return require "pl".template

types/pl/test.d.tl

@@ -0,0 +1 @@
+return require "pl".test

types/pl/text.d.tl

@@ -0,0 +1 @@
+return require "pl".text

types/pl/types.d.tl

@@ -0,0 +1 @@
+return require "pl".types

types/pl/url.d.tl

@@ -0,0 +1 @@
+return require "pl".url

types/pl/utils.d.tl

@@ -0,0 +1 @@
+return require "pl".utils

types/pl/xml.d.tl

@@ -0,0 +1 @@
+return require "pl".xml

types/socket.d.tl

@@ -0,0 +1,158 @@
+local ltn12 = require("ltn12")
+local Sink = ltn12.Sink
+local Source = ltn12.Source
+
+local record socket
+   record TCP
+      -- master methods
+      bind: function(TCP, string, integer)
+      connect: function(TCP, string, integer): integer, string
+      listen: function(TCP, integer): integer, string
+
+      -- client methods
+      getpeername: function(TCP): string, integer
+
+      enum TCPReceivePattern
+         "*l"
+         "*a"
+      end
+      enum TCPReceiveError
+         "closed"
+         "timeout"
+      end
+      receive: function(TCP, TCPReceivePattern|integer, string): string, TCPReceiveError
+
+      send: function(TCP, string, integer, integer): integer, string, integer
+
+      enum TCPShutdownMode
+         "both"
+         "send"
+         "receive"
+      end
+      shutdown: function(TCP, TCPShutdownMode): integer
+
+      -- server methods
+      accept: function(TCP): TCP, string
+
+      -- client and server methods
+      enum TCPOption
+         "keepalive"
+         "reuseaddr"
+         "tcp-nodelay"
+      end
+      enum TCPLinger
+         "linger"
+      end
+      record TCPLingerOption
+         on: boolean
+         timeout: integer
+      end
+      setoption: function(TCP, TCPOption): integer
+      setoption: function(TCP, TCPLinger, TCPLingerOption): integer
+
+      -- master, client, and server methods
+      close: function(TCP)
+
+      getsockname: function(TCP): string, integer
+
+      getstats: function(TCP): integer, integer, integer
+
+      setstats: function(TCP, integer, integer, integer): integer
+
+      enum TCPTimeoutMode
+         "b"
+         "t"
+      end
+      settimeout: function(TCP, integer, TCPTimeoutMode)
+   end
+   record UDP
+      close: function(UDP)
+
+      getpeername: function(UDP): string, integer
+
+      getsockname: function(UDP): string, integer
+
+      enum UDPTimeout
+         "timeout"
+      end
+      receive: function(UDP, integer): string, UDPTimeout
+
+      receivefrom: function(UDP, integer): string, string, integer, UDPTimeout
+
+      send: function(UDP, string): integer, string
+
+      sendto: function(UDP, string, string, integer): integer, string
+
+      setpeername: function(UDP, string, integer): integer, string
+
+      setsockname: function(UDP, string, integer): integer, string
+
+      enum UDPOptions
+         "dontroute"
+         "broadcast"
+      end
+      setoption: function(UDP, UDPOptions, boolean): integer, string
+
+      settimeout: function(UDP, integer)
+   end
+   tcp: function(): TCP, string
+
+   udp: function(): UDP, string
+
+   record dns
+      record DNSResolved
+         name: string
+         alias: {string}
+         ip: {string}
+      end
+      toip: function(): string
+      tohostname: function(string): string, DNSResolved|string
+      gethostname: function(string): string, DNSResolved|string
+   end
+
+   bind: function(string, integer, integer): TCP
+
+   connect: function(string, integer, string, integer): TCP
+
+   _DEBUG: boolean
+
+   newtry: function(function): function
+
+   protect: function(function): function
+
+   -- tagged records/Table Union types would be nice here,
+   -- as this should be {TCP|UDP}
+   -- but I imagine this should be fine for most uses
+   select: function({UDP}, {UDP}, integer): {UDP}, {UDP}, string
+   select: function({TCP}, {TCP}, integer): {TCP}, {TCP}, string
+
+   enum SinkMode
+      "http-chunked"
+      "close-when-done"
+      "keep-open"
+   end
+
+   sink: function(SinkMode, UDP): Sink
+   sink: function(SinkMode, TCP): Sink
+
+   skip: function(integer, ...: any): any...
+
+   sleep: function(integer)
+
+   enum SourceMode
+      "http-chunked"
+      "by-length"
+      "until-closed"
+   end
+
+   source: function(SourceMode, TCP, integer): Source
+   source: function(SourceMode, UDP, integer): Source
+
+   gettime: function(): integer
+
+   try: function(...: any): any...
+
+   _VERSION: string
+end
+
+return socket

types/socket/ftp.d.tl

@@ -0,0 +1,39 @@
+local ltn12 = require("ltn12")
+local type Pump = ltn12.Pump
+local type Sink = ltn12.Sink
+
+local record ftp
+   get: function(string): string, string
+   record FTPGetInfo
+      host: string
+      sink: Sink
+      argument: string
+      path: string
+      user: string
+      password: string
+      command: string
+      port: integer
+      type: string
+      step: Pump
+      create: function()
+   end
+   get: function(FTPGetInfo): integer, string
+
+   record FTPPutInfo
+      host: string
+      source: Sink -- yes, this is a sink
+      argument: string
+      path: string
+      user: string
+      password: string
+      command: string
+      port: integer
+      type: string
+      step: Pump
+      create: function()
+   end
+   put: function(string, string): integer, string
+   put: function(FTPPutInfo): integer, string
+end
+
+return ftp

types/socket/http.d.tl

@@ -0,0 +1,28 @@
+local ltn12 = require("ltn12")
+local type Pump = ltn12.Pump
+local type Sink = ltn12.Sink
+local type Source = ltn12.Source
+
+local record http
+   request: function(string): string, integer|string, string, string
+   request: function(string, string): string, integer|string, string, string
+   record HTTPRequest
+      url: string
+      sink: Sink
+      method: string
+      headers: {string:string}
+      source: Source
+      step: Pump
+      proxy: string
+      redirect: boolean
+      create: function
+   end
+   request: function(HTTPRequest): string, integer|string, string, string
+
+   PORT: integer
+   PROXY: string
+   TIMEOUT: integer
+   USERAGENT: string
+end
+
+return http

types/socket/smtp.d.tl

@@ -0,0 +1,32 @@
+local ltn12 = require("ltn12")
+local type Pump = ltn12.Pump
+local type Source = ltn12.Source
+
+local record smtp
+   record Message
+      headers: {string:string}
+      body: Source | string | MultipartMessage
+   end
+   record MultipartMessage
+      {Message}
+      preamble: string
+      epilogue: string
+   end
+   message: function(Message): Source
+
+   record SMTPSendFormat
+      from: string
+      rcpt: string | {string}
+      source: Source
+      user: string
+      password: string
+      server: string
+      port: integer
+      domain: string
+      step: Pump
+      create: function
+   end
+   send: function(SMTPSendFormat): integer, string
+end
+
+return smtp

types/socket/url.d.tl

@@ -0,0 +1,25 @@
+local record url
+   record ParsedUrl
+      url: string
+      scheme: string
+      authority: string
+      path: string
+      params: string
+      query: string
+      fragment: string
+      userinfo: string
+      host: string
+      port: string
+      user: string
+      password: string
+   end
+
+   absolute: function(string, string): string
+   build: function(ParsedUrl): string
+   build_path: function({string}, any): string
+   escape: function(string): string
+   parse: function(string, table): ParsedUrl
+   parse_path: function(string): {string}
+   unescape: function(string): string
+end
+return url

types/web_sanitize.d.tl

@@ -0,0 +1,16 @@
+local record Web_Sanitize
+   sanitize_html: function(unsafe_html: string): string
+   extract_text: function(unsafe_html: string): string
+   sanitize_style: function(unsafe_style_attributes: string): string
+
+   record whitelist
+      clone: function(): whitelist
+      tags: table
+      add_attributes: table
+      self_closing: table
+   end
+
+   Sanitizer: function(parameters: table): function(unsafe_html: string): string
+end
+
+return Web_Sanitize

types/web_sanitize/query/scan_html.d.tl

@@ -0,0 +1,23 @@
+local record Scanner
+   record HTMLNode
+    tag: string
+    type: string | nil
+    num: number
+    self_closing: boolean | nil
+    attr: table
+    outer_html: function(self: HTMLNode): string
+    inner_html: function(self: HTMLNode): string
+    inner_text: function(self: HTMLNode): string
+    -- TODO : add replacement methods
+   end
+
+   record NodeStack
+      -- stack[n] - get the nth item in the stack (as an HTMLNode)
+      current: function(self: NodeStack): HTMLNode
+      is: function(self: NodeStack, query: string): boolean
+   end
+
+   scan_html: function(html_text: string, callback: function(stack: NodeStack), opts: table)
+end
+
+return Scanner