great refactoring: now also prettifies code in Markdown documents

html/ldoc.ltp

@@ -16,20 +16,12 @@
 	<div id="product_description"></div>
 </div> <!-- id="product" -->
 
-# local function use_li(ls)
-#   if #ls > 1 then return '<li>','</li>' else return '','' end
-# end
-# local function display_name(item)
-#   if item.type == 'function' then return item.name..'&nbsp;'..item.args
-#   else return item.name end
-#  end
-# local function no_spaces(s) return (s:gsub('%s','_')) end
-# local function title(s) return (s:gsub('(%a)(%a*)',function(f,r)
-#     return f:upper()..r
-# end)) end
 
 <div id="main">
 
+# local no_spaces = ldoc.no_spaces
+# local use_li = ldoc.use_li
+# local display_name = ldoc.display_name
 # local iter = ldoc.modules.iter
 # local M = ldoc.markup
 
@@ -87,7 +79,7 @@
 <div id="content">
 
 #if module then
-<h1>$(title(module.type)) <code>$(module.name)</code></h1>
+<h1>$(ldoc.titlecase(module.type)) <code>$(module.name)</code></h1>
 # end
 
 # if ldoc.body then -- verbatim HTML as contents; 'non-code' entries

ldoc.lua

@@ -7,7 +7,7 @@
 require 'pl'
 
 local append = table.insert
-local template = require 'pl.template'
+
 local lapp = require 'pl.lapp'
 
 -- so we can find our private modules
@@ -35,14 +35,16 @@ ldoc, a documentation generator for Lua, vs 0.5
   <file> (string) source file or directory containing source
 ]]
 
-local lexer = require 'ldoc.lexer'
 local doc = require 'ldoc.doc'
 local lang = require 'ldoc.lang'
-local Item,File,Module = doc.Item,doc.File,doc.Module
 local tools = require 'ldoc.tools'
 local global = require 'builtin.globals'
 local markup = require 'ldoc.markup'
+local parse = require 'ldoc.parse'
 local KindMap = tools.KindMap
+local Item,File,Module = doc.Item,doc.File,doc.Module
+local quit = utils.quit
+
 
 class.ModuleMap(KindMap)
 
@@ -70,6 +72,17 @@ ProjectMap:add_kind('script','Scripts')
 ProjectMap:add_kind('topic','Topics')
 ProjectMap:add_kind('example','Examples')
 
+local lua, cc = lang.lua, lang.cc
+
+local file_types = {
+   ['.lua'] = lua,
+   ['.ldoc'] = lua,
+   ['.luadoc'] = lua,
+   ['.c'] = cc,
+   ['.cpp'] = cc,
+   ['.cxx'] = cc,
+   ['.C'] = cc
+}
 
 ------- ldoc external API ------------
 
@@ -83,7 +96,9 @@ function ldoc.alias (a,tag)
 end
 
 function ldoc.add_language_extension(ext,lang)
-   add_language_extension(ext,lang)
+   lang = (lang=='c' and cc) or (lang=='lua' and lua) or quit('unknown language')
+   if ext:sub(1,1) ~= '.' then ext = '.'..ext end
+   file_types[ext] = lang
 end
 
 function ldoc.add_section (name,title,subname)
@@ -128,196 +143,6 @@ local function quote (s)
    return "'"..s.."'"
 end
 
------- Parsing the Source --------------
--- This uses the lexer from PL, but it should be possible to use Peter Odding's
--- excellent Lpeg based lexer instead.
-
-local tnext = lexer.skipws
-
--- a pattern particular to LuaDoc tag lines: the line must begin with @TAG,
--- followed by the value, which may extend over several lines.
-local luadoc_tag = '^%s*@(%a+)%s(.+)'
-
--- assumes that the doc comment consists of distinct tag lines
-function parse_tags(text)
-   local lines = stringio.lines(text)
-   local preamble, line = tools.grab_while_not(lines,luadoc_tag)
-   local tag_items = {}
-   local follows
-   while line do
-      local tag,rest = line:match(luadoc_tag)
-      follows, line = tools.grab_while_not(lines,luadoc_tag)
-      append(tag_items,{tag, rest .. '\n' .. follows})
-   end
-   return preamble,tag_items
-end
-
--- This takes the collected comment block, and uses the docstyle to
--- extract tags and values.  Assume that the summary ends in a period or a question
--- mark, and everything else in the preamble is the description.
--- If a tag appears more than once, then its value becomes a list of strings.
--- Alias substitution and @TYPE NAME shortcutting is handled by Item.check_tag
-local function extract_tags (s)
-   if s:match '^%s*$' then return {} end
-   local preamble,tag_items = parse_tags(s)
-   local strip = tools.strip
-   local summary,description = preamble:match('^(.-[%.?])%s(.+)')
-   if not summary then summary = preamble end  --  and strip(description) ?
-   local tags = {summary=summary and strip(summary),description=description}
-   for _,item in ipairs(tag_items) do
-      local tag,value = item[1],item[2]
-      tag = Item.check_tag(tags,tag)
-      value = strip(value)
-      local old_value = tags[tag]
-      if old_value then
-         if type(old_value)=='string' then tags[tag] = List{old_value} end
-         tags[tag]:append(value)
-      else
-         tags[tag] = value
-      end
-   end
-   return Map(tags)
-end
-
-local quit = utils.quit
-
-
--- parses a Lua or C file, looking for ldoc comments. These are like LuaDoc comments;
--- they start with multiple '-'. (Block commments are allowed)
--- If they don't define a name tag, then by default
--- it is assumed that a function definition follows. If it is the first comment
--- encountered, then ldoc looks for a call to module() to find the name of the
--- module if there isn't an explicit module name specified.
-
-local function parse_file(fname,lang)
-   local line,f = 1
-   local F = File(fname)
-   local module_found, first_comment = false,true
-
-   local tok,f = lang.lexer(fname)
-   local toks = tools.space_skip_getter(tok)
-
-    function lineno ()
-        while true do
-            local res = lexer.lineno(tok)
-            if type(res) == 'number' then return res end
-            if res == nil then return nil end
-        end
-    end
-   function filename () return fname end
-
-   function F:warning (msg,kind)
-      kind = kind or 'warning'
-      lineno() -- why is this necessary?
-      lineno()
-      io.stderr:write(kind..' '..fname..':'..lineno()..' '..msg,'\n')
-   end
-
-   function F:error (msg)
-      self:warning(msg,'error')
-      os.exit(1)
-   end
-
-   local function add_module(tags,module_found,old_style)
-      tags.name = module_found
-      tags.class = 'module'
-      local item = F:new_item(tags,lineno())
-      item.old_style = old_style
-   end
-
-   local t,v = tok()
-   while t do
-      if t == 'comment' then
-         local comment = {}
-         local ldoc_comment,block = lang:start_comment(v)
-         if ldoc_comment and block then
-            t,v = lang:grab_block_comment(v,tok)
-         end
-
-         if lang:empty_comment(v)  then -- ignore rest of empty start comments
-            t,v = tok()
-         end
-
-         while t and t == 'comment' do
-            v = lang:trim_comment(v)
-            append(comment,v)
-            t,v = tok()
-            if t == 'space' and not v:match '\n' then
-               t,v = tok()
-            end
-         end
-         if not t then break end -- no more file!
-
-         if t == 'space' then t,v = tnext(tok) end
-
-         local fun_follows, tags, is_local
-         if ldoc_comment or first_comment then
-            comment = table.concat(comment)
-            if not ldoc_comment and first_comment then
-               F:warning("first comment must be a doc comment!")
-               break
-            end
-            first_comment = false
-            fun_follows, is_local = lang:function_follows(t,v,tok)
-            if fun_follows or comment:find '@'then
-               tags = extract_tags(comment)
-               if doc.project_level(tags.class) then
-                  module_found = tags.name
-               end
-               if tags.class == 'function' then
-                  fun_follows, is_local = false, false
-               end
-            end
-         end
-         -- some hackery necessary to find the module() call
-         if not module_found and ldoc_comment then
-            local old_style
-            module_found,t,v = lang:find_module(tok,t,v)
-            -- right, we can add the module object ...
-            old_style = module_found ~= nil
-            if not module_found or module_found == '...' then
-               if not t then quit(fname..": end of file") end -- run out of file!
-               -- we have to guess the module name
-               module_found = tools.this_module_name(args.package,fname)
-            end
-            if not tags then tags = extract_tags(comment) end
-            add_module(tags,module_found,old_style)
-            tags = nil
-            -- if we did bump into a doc comment, then we can continue parsing it
-         end
-
-         -- end of a block of document comments
-         if ldoc_comment and tags then
-            local line = t ~= nil and lineno() or 666
-            if t ~= nil then
-               if fun_follows then -- parse the function definition
-                  lang:parse_function_header(tags,tok,toks)
-               else
-                  lang:parse_extra(tags,tok,toks)
-               end
-            end
-            -- local functions treated specially
-            if tags.class == 'function' and (is_local or tags['local']) then
-               tags.class = 'lfunction'
-            end
-            if tags.name then
-               F:new_item(tags,line).inferred = fun_follows
-            end
-            if not t then break end
-         end
-      end
-      if t ~= 'comment' then t,v = tok() end
-   end
-   if f then f:close() end
-   return F
-end
-
-function read_file(name,lang)
-   local F = parse_file(name,lang)
-   F:finish()
-   return F
-end
-
 --- processing command line and preparing for output ---
 
 local F
@@ -399,23 +224,6 @@ local function setup_package_base()
    end
 end
 
-local lua, cc = lang.lua, lang.cc
-
-local file_types = {
-   ['.lua'] = lua,
-   ['.ldoc'] = lua,
-   ['.luadoc'] = lua,
-   ['.c'] = cc,
-   ['.cpp'] = cc,
-   ['.cxx'] = cc,
-   ['.C'] = cc
-}
-
-function add_language_extension (ext,lang)
-   lang = (lang=='c' and cc) or (lang=='lua' and lua) or quit('unknown language')
-   if ext:sub(1,1) ~= '.' then ext = '.'..ext end
-   file_types[ext] = lang
-end
 
 --------- processing files ---------------------
 -- ldoc may be given a file, or a directory. `args.file` may also be specified in config.ld
@@ -427,7 +235,8 @@ local function process_file (f, file_list)
    local ftype = file_types[ext]
    if ftype then
       if args.verbose then print(path.basename(f)) end
-      local F = read_file(f,ftype)
+      local F,err = parse.file(f,ftype)
+      if err then quit(err) end
       file_list:append(F)
    end
 end
@@ -504,6 +313,8 @@ end
 
 if type(ldoc.examples) == 'table' then
    local prettify = require 'ldoc.prettify'
+   local formatter = markup.create(ldoc,'plain')
+   prettify.resolve_inline_references = markup.resolve_inline_references
 
    local function process_example (f, file_list)
       local item = add_special_project_entity(f,{
@@ -581,7 +392,7 @@ if args.filter ~= 'none' then
    os.exit()
 end
 
-local css, templ = 'ldoc.css','ldoc.ltp'
+ldoc.css, ldoc.templ = 'ldoc.css','ldoc.ltp'
 
 local function style_dir (sname)
    local style = ldoc[sname]
@@ -624,7 +435,7 @@ if not args.ext:find '^%.' then
 end
 
 if args.one then
-   css = 'ldoc_one.css'
+   ldoc.css = 'ldoc_one.css'
 end
 
 -- '!' here means 'use same directory as ldoc.lua
@@ -632,120 +443,21 @@ local ldoc_html = path.join(ldoc_dir,'html')
 if args.style == '!' then args.style = ldoc_html end
 if args.template == '!' then args.template = ldoc_html end
 
-local module_template,err = utils.readfile (path.join(args.template,templ))
-if not module_template then
-   quit("template not found. Use -l to specify directory containing ldoc.ltp")
-end
 
 -- create the function that renders text (descriptions and summaries)
 ldoc.markup = markup.create(ldoc, args.format)
 
--- this generates the internal module/function references; strictly speaking,
--- it should be (and was) part of the template, but inline references in
--- Markdown required it be more widely available. A temporary situation!
 
-function ldoc.href(see)
+ldoc.single = not multiple_files and first_module or nil
-   if see.href then -- explict reference, e.g. to Lua manual
+ldoc.log = print
-      return see.href
+ldoc.kinds = project
-   else
+ldoc.modules = module_list
-      return ldoc.ref_to_module(see.mod)..'#'..see.name
+ldoc.title = ldoc.title or args.title
-   end
+ldoc.project = ldoc.project or args.project
-end
 
--- this is either called from the 'root' (index or single module) or
+local html = require 'ldoc.html'
--- from the 'modules' etc directories. If we are in one of those directories,
--- then linking to another kind is `../kind/name`; to the same kind is just `name`.
--- If we are in the root, then it is `kind/name`.
 
-function ldoc.ref_to_module (mod)
+html.generate_output(ldoc, args, project)
-   local base = "" -- default: same directory
-   local kind, module = mod.kind, ldoc.module
-   local name = mod.name -- default: name of module
-   if not ldoc.single then
-      if module then -- we are in kind/
-         if module.type ~= type then -- cross ref to ../kind/
-            base = "../"..kind.."/"
-         end
-      else -- we are in root: index
-         base = kind..'/'
-      end
-   else -- single module
-      if mod == first_module then
-         name = ldoc.output
-         if not ldoc.root then base = '../' end
-      elseif ldoc.root then -- ref to other kinds (like examples)
-         base = kind..'/'
-      else
-         if module.type ~= type then -- cross ref to ../kind/
-            base = "../"..kind.."/"
-         end
-      end
-   end
-   return base..name..'.html'
-end
-
-
-local function generate_output()
-   local check_directory, check_file, writefile = tools.check_directory, tools.check_file, tools.writefile
-   ldoc.single = not multiple_files and first_module or nil
-   ldoc.log = print
-   ldoc.kinds = project
-   ldoc.css = css
-   ldoc.modules = module_list
-   ldoc.title = ldoc.title or args.title
-   ldoc.project = ldoc.project or args.project
-
-   -- in single mode there is one module and the 'index' is the
-   -- documentation for that module.
-   ldoc.module = ldoc.single and first_module or nil
-   ldoc.root = true
-   local out,err = template.substitute(module_template,{
-      ldoc = ldoc,
-      module = ldoc.module
-    })
-   ldoc.root = false
-   if not out then quit("template failed: "..err) end
-
-   check_directory(args.dir) -- make sure output directory is ok
-
-   args.dir = args.dir .. path.sep
-
-   check_file(args.dir..css, path.join(args.style,css)) -- has CSS been copied?
-
-   -- write out the module index
-   writefile(args.dir..args.output..args.ext,out)
-
-   -- write out the per-module documentation
-   -- in single mode, we exclude any modules since the module has been done;
-   -- this step is then only for putting out any examples or topics
-   ldoc.css = '../'..css
-   ldoc.output = args.output
-   for kind, modules in project() do
-      kind = kind:lower()
-      if not ldoc.single or ldoc.single and kind ~= 'modules' then
-         check_directory(args.dir..kind)
-         for m in modules() do
-            ldoc.module = m
-            ldoc.body = m.body
-            if ldoc.body then
-               ldoc.body = m.postprocess(ldoc.body)
-            end
-            out,err = template.substitute(module_template,{
-               module=m,
-               ldoc = ldoc
-            })
-            if not out then
-               quit('template failed for '..m.name..': '..err)
-            else
-               writefile(args.dir..kind..'/'..m.name..args.ext,out)
-            end
-         end
-      end
-   end
-   if not args.quiet then print('output written to '..args.dir) end
-end
-
-generate_output()
 
 if args.verbose then
    print 'modules'

ldoc/doc.lua

@@ -173,6 +173,24 @@ function File:finish()
    end
 end
 
+-- some serious hackery. We force sections into this 'module',
+-- and ensure that there is a dummy item so that the section
+-- is not empty.
+
+function File:add_document_section(title)
+   local section = title:gsub('%A','_')
+   self:new_item {
+      name = section,
+      class = 'section',
+      summary = title
+   }
+   self:new_item {
+      name = 'dumbo',
+      class = 'function',
+   }
+   return section
+end
+
 function Item:_init(tags,file,line)
    self.file = file
    self.lineno = line

ldoc/html.lua

@@ -0,0 +1,142 @@
+------ generating HTML output ---------
+-- Although this can be generalized for outputting any format, since the template
+-- is language-agnostic, this implementation concentrates on HTML.
+-- This does the actual generation of HTML, and provides support functions in the ldoc
+-- table for the template
+--
+-- A fair amount of the complexity comes from operating in two basic modes; first, where
+-- there is a number of modules (classic LuaDoc) or otherwise, where there is only one
+-- module and the index contains the documentation for that module.
+--
+-- Like LuaDoc, LDoc puts similar kinds of documentation files in their own directories.
+-- So module docs go into 'modules/', scripts go into 'scripts/', and so forth. LDoc
+-- generalizes the idea of these project-level categories and in fact custom categories
+-- can be created (refered to as 'kinds' in the code)
+
+local template = require 'pl.template'
+local tools = require 'ldoc.tools'
+
+local html = {}
+
+
+local quit = utils.quit
+
+function html.generate_output(ldoc, args, project)
+   local check_directory, check_file, writefile = tools.check_directory, tools.check_file, tools.writefile
+
+
+   -- this generates the internal module/function references
+   function ldoc.href(see)
+      if see.href then -- explict reference, e.g. to Lua manual
+         return see.href
+      else
+         return ldoc.ref_to_module(see.mod)..'#'..see.name
+      end
+   end
+
+   -- this is either called from the 'root' (index or single module) or
+   -- from the 'modules' etc directories. If we are in one of those directories,
+   -- then linking to another kind is `../kind/name`; to the same kind is just `name`.
+   -- If we are in the root, then it is `kind/name`.
+   function ldoc.ref_to_module (mod)
+      local base = "" -- default: same directory
+      local kind, module = mod.kind, ldoc.module
+      local name = mod.name -- default: name of module
+      if not ldoc.single then
+         if module then -- we are in kind/
+            if module.type ~= type then -- cross ref to ../kind/
+               base = "../"..kind.."/"
+            end
+         else -- we are in root: index
+            base = kind..'/'
+         end
+      else -- single module
+         if mod == ldoc.single then
+            name = ldoc.output
+            if not ldoc.root then base = '../' end
+         elseif ldoc.root then -- ref to other kinds (like examples)
+            base = kind..'/'
+         else
+            if module.type ~= type then -- cross ref to ../kind/
+               base = "../"..kind.."/"
+            end
+         end
+      end
+      return base..name..'.html'
+   end
+
+   function ldoc.use_li(ls)
+      if #ls > 1 then return '<li>','</li>' else return '','' end
+   end
+
+   function ldoc.display_name(item)
+      if item.type == 'function' then return item.name..'&nbsp;'..item.args
+      else return item.name end
+   end
+
+   function ldoc.no_spaces(s) return (s:gsub('%s','_')) end
+
+   function ldoc.titlecase(s)
+      return (s:gsub('(%a)(%a*)',function(f,r)
+         return f:upper()..r
+      end))
+   end
+
+   local module_template,err = utils.readfile (path.join(args.template,ldoc.templ))
+   if not module_template then
+      quit("template not found. Use -l to specify directory containing ldoc.ltp")
+   end
+
+   local css = ldoc.css
+
+   -- in single mode there is one module and the 'index' is the
+   -- documentation for that module.
+   ldoc.module = ldoc.single
+   ldoc.root = true
+   local out,err = template.substitute(module_template,{
+      ldoc = ldoc,
+      module = ldoc.module
+    })
+   ldoc.root = false
+   if not out then quit("template failed: "..err) end
+
+   check_directory(args.dir) -- make sure output directory is ok
+
+   args.dir = args.dir .. path.sep
+
+   check_file(args.dir..css, path.join(args.style,css)) -- has CSS been copied?
+
+   -- write out the module index
+   writefile(args.dir..args.output..args.ext,out)
+
+   -- write out the per-module documentation
+   -- in single mode, we exclude any modules since the module has been done;
+   -- this step is then only for putting out any examples or topics
+   ldoc.css = '../'..css
+   ldoc.output = args.output
+   for kind, modules in project() do
+      kind = kind:lower()
+      if not ldoc.single or ldoc.single and kind ~= 'modules' then
+         check_directory(args.dir..kind)
+         for m in modules() do
+            ldoc.module = m
+            ldoc.body = m.body
+            if ldoc.body then
+               ldoc.body = m.postprocess(ldoc.body)
+            end
+            out,err = template.substitute(module_template,{
+               module=m,
+               ldoc = ldoc
+            })
+            if not out then
+               quit('template failed for '..m.name..': '..err)
+            else
+               writefile(args.dir..kind..'/'..m.name..args.ext,out)
+            end
+         end
+      end
+   end
+   if not args.quiet then print('output written to '..args.dir) end
+end
+
+return html

ldoc/markup.lua

@@ -6,15 +6,17 @@
 require 'pl'
 local doc = require 'ldoc.doc'
 local utils = require 'pl.utils'
-local quit = utils.quit
+local prettify = require 'ldoc.prettify'
+local quit, concat, lstrip = utils.quit, table.concat, stringx.lstrip
 local markup = {}
 
 -- workaround Markdown's need for blank lines around indented blocks
 -- (does mean you have to keep indentation discipline!)
 function markup.insert_markdown_lines (txt)
    local res, append = {}, table.insert
-   local last_indent, start_indent, skip = -1, -1, false
+   local last_indent, start_indent, skip, code = -1, -1, false, nil
    for line in stringx.lines(txt) do
+      line = line:gsub('\t','    ')  -- some people like tabs ;)
       if not line:match '^%s*$' then --ignore blank lines
          local indent = #line:match '^%s*'
          if start_indent < 0 then -- initialize indents at start
@@ -24,49 +26,53 @@ function markup.insert_markdown_lines (txt)
          if indent < start_indent then -- end of indented block
             append(res,'')
             skip = false
+            if code then
+               code = concat(code,'\n')
+               code, err = prettify.lua(code)
+               if code then
+                  append(res,code)
+                  append(res,'</pre>')
+               end
+               code = nil
+            end
          end
          if not skip and indent > last_indent then -- start of indent
             append(res,'')
             skip = true
             start_indent = indent
+            if indent >= 4 then
+               code = {}
             end
-         append(res,line)
+         end
-         last_indent = indent
+         if code then
+            append(code, line:sub(start_indent))
          else
+            append(res,line)
+         end
+         last_indent = indent
+      elseif not code then
          append(res,'')
       end
    end
-   return table.concat(res,'\n')
+   res = concat(res,'\n')
+   return res
 end
 
 -- for readme text, the idea here is to insert module sections at ## so that
 -- they can appear in the contents list as a ToC
 function markup.add_sections(F, txt)
    local res, append = {}, table.insert
-   local last_indent, start_indent, skip = -1, -1, false
    for line in stringx.lines(txt) do
       local title = line:match '^##[^#]%s*(.+)'
       if title then
-         -- some serious hackery. We force sections into this 'module',
+         local section = F:add_document_section(title)
-         -- and ensure that there is a dummy item so that the section
-         -- is not empty.
-         local section = title:gsub('%A','_')
-         F:new_item {
-            name = section,
-            class = 'section',
-            summary = title
-         }
-         F:new_item {
-            name = 'dumbo',
-            class = 'function',
-         }
          append(res,('<a id="%s"></a>\n'):format(section))
          append(res,line)
       else
          append(res,line)
       end
    end
-   return table.concat(res,'\n')
+   return concat(res,'\n')
 end
 
 local function handle_reference (ldoc, name)
@@ -104,6 +110,7 @@ function markup.create (ldoc, format)
    markup.href = function(ref)
       return ldoc.href(ref)
    end
+
    if format == 'plain' then
       processor = function(txt)
          if txt == nil then return '' end
@@ -128,6 +135,7 @@ function markup.create (ldoc, format)
       return resolve_inline_references(ldoc, txt)
    end
    markup.processor = processor
+   prettify.resolve_inline_references = markup.resolve_inline_references
    return processor
 end
 

ldoc/parse.lua

@@ -0,0 +1,201 @@
+-- parsing code for doc comments
+
+require 'pl'
+local lexer = require 'ldoc.lexer'
+local tools = require 'ldoc.tools'
+local doc = require 'ldoc.doc'
+local Item,File = doc.Item,doc.File
+
+------ Parsing the Source --------------
+-- This uses the lexer from PL, but it should be possible to use Peter Odding's
+-- excellent Lpeg based lexer instead.
+
+local parse = {}
+
+local tnext, append = lexer.skipws, table.insert
+
+-- a pattern particular to LuaDoc tag lines: the line must begin with @TAG,
+-- followed by the value, which may extend over several lines.
+local luadoc_tag = '^%s*@(%a+)%s(.+)'
+
+-- assumes that the doc comment consists of distinct tag lines
+function parse_tags(text)
+   local lines = stringio.lines(text)
+   local preamble, line = tools.grab_while_not(lines,luadoc_tag)
+   local tag_items = {}
+   local follows
+   while line do
+      local tag,rest = line:match(luadoc_tag)
+      follows, line = tools.grab_while_not(lines,luadoc_tag)
+      append(tag_items,{tag, rest .. '\n' .. follows})
+   end
+   return preamble,tag_items
+end
+
+-- This takes the collected comment block, and uses the docstyle to
+-- extract tags and values.  Assume that the summary ends in a period or a question
+-- mark, and everything else in the preamble is the description.
+-- If a tag appears more than once, then its value becomes a list of strings.
+-- Alias substitution and @TYPE NAME shortcutting is handled by Item.check_tag
+local function extract_tags (s)
+   if s:match '^%s*$' then return {} end
+   local preamble,tag_items = parse_tags(s)
+   local strip = tools.strip
+   local summary,description = preamble:match('^(.-[%.?])%s(.+)')
+   if not summary then summary = preamble end  --  and strip(description) ?
+   local tags = {summary=summary and strip(summary),description=description}
+   for _,item in ipairs(tag_items) do
+      local tag,value = item[1],item[2]
+      tag = Item.check_tag(tags,tag)
+      value = strip(value)
+      local old_value = tags[tag]
+      if old_value then
+         if type(old_value)=='string' then tags[tag] = List{old_value} end
+         tags[tag]:append(value)
+      else
+         tags[tag] = value
+      end
+   end
+   return Map(tags)
+end
+
+
+
+-- parses a Lua or C file, looking for ldoc comments. These are like LuaDoc comments;
+-- they start with multiple '-'. (Block commments are allowed)
+-- If they don't define a name tag, then by default
+-- it is assumed that a function definition follows. If it is the first comment
+-- encountered, then ldoc looks for a call to module() to find the name of the
+-- module if there isn't an explicit module name specified.
+
+local function parse_file(fname,lang)
+   local line,f = 1
+   local F = File(fname)
+   local module_found, first_comment = false,true
+
+   local tok,f = lang.lexer(fname)
+   local toks = tools.space_skip_getter(tok)
+
+    function lineno ()
+        while true do
+            local res = lexer.lineno(tok)
+            if type(res) == 'number' then return res end
+            if res == nil then return nil end
+        end
+    end
+   function filename () return fname end
+
+   function F:warning (msg,kind)
+      kind = kind or 'warning'
+      lineno() -- why is this necessary?
+      lineno()
+      io.stderr:write(kind..' '..fname..':'..lineno()..' '..msg,'\n')
+   end
+
+   function F:error (msg)
+      self:warning(msg,'error')
+      os.exit(1)
+   end
+
+   local function add_module(tags,module_found,old_style)
+      tags.name = module_found
+      tags.class = 'module'
+      local item = F:new_item(tags,lineno())
+      item.old_style = old_style
+   end
+
+   local t,v = tok()
+   while t do
+      if t == 'comment' then
+         local comment = {}
+         local ldoc_comment,block = lang:start_comment(v)
+         if ldoc_comment and block then
+            t,v = lang:grab_block_comment(v,tok)
+         end
+
+         if lang:empty_comment(v)  then -- ignore rest of empty start comments
+            t,v = tok()
+         end
+
+         while t and t == 'comment' do
+            v = lang:trim_comment(v)
+            append(comment,v)
+            t,v = tok()
+            if t == 'space' and not v:match '\n' then
+               t,v = tok()
+            end
+         end
+         if not t then break end -- no more file!
+
+         if t == 'space' then t,v = tnext(tok) end
+
+         local fun_follows, tags, is_local
+         if ldoc_comment or first_comment then
+            comment = table.concat(comment)
+            if not ldoc_comment and first_comment then
+               F:warning("first comment must be a doc comment!")
+               break
+            end
+            first_comment = false
+            fun_follows, is_local = lang:function_follows(t,v,tok)
+            if fun_follows or comment:find '@'then
+               tags = extract_tags(comment)
+               if doc.project_level(tags.class) then
+                  module_found = tags.name
+               end
+               if tags.class == 'function' then
+                  fun_follows, is_local = false, false
+               end
+            end
+         end
+         -- some hackery necessary to find the module() call
+         if not module_found and ldoc_comment then
+            local old_style
+            module_found,t,v = lang:find_module(tok,t,v)
+            -- right, we can add the module object ...
+            old_style = module_found ~= nil
+            if not module_found or module_found == '...' then
+               if not t then return nil, fname..": end of file" end -- run out of file!
+               -- we have to guess the module name
+               module_found = tools.this_module_name(args.package,fname)
+            end
+            if not tags then tags = extract_tags(comment) end
+            add_module(tags,module_found,old_style)
+            tags = nil
+            -- if we did bump into a doc comment, then we can continue parsing it
+         end
+
+         -- end of a block of document comments
+         if ldoc_comment and tags then
+            local line = t ~= nil and lineno() or 666
+            if t ~= nil then
+               if fun_follows then -- parse the function definition
+                  lang:parse_function_header(tags,tok,toks)
+               else
+                  lang:parse_extra(tags,tok,toks)
+               end
+            end
+            -- local functions treated specially
+            if tags.class == 'function' and (is_local or tags['local']) then
+               tags.class = 'lfunction'
+            end
+            if tags.name then
+               F:new_item(tags,line).inferred = fun_follows
+            end
+            if not t then break end
+         end
+      end
+      if t ~= 'comment' then t,v = tok() end
+   end
+   if f then f:close() end
+   return F
+end
+
+function parse.file(name,lang)
+   local F,err = parse_file(name,lang)
+   if err then return nil,err end
+   F:finish()
+   return F
+end
+
+return parse

ldoc/prettify.lua

@@ -5,7 +5,6 @@
 -- `@{example:test-fun}`.
 require 'pl'
 local lexer = require 'ldoc.lexer'
-local markup = require 'ldoc.markup'
 local tnext = lexer.skipws
 local prettify = {}
 
@@ -24,11 +23,6 @@ local function span(t,val)
    return ('<span class="%s">%s</span>'):format(t,val)
 end
 
-local function link(file,ref,text)
-   text = text or ref
-   return ('<a class="L" href="%s.html#%s">%s</a>'):format(file,ref,text)
-end
-
 local spans = {keyword=true,number=true,string=true,comment=true}
 
 function prettify.lua (code)
@@ -43,7 +37,7 @@ function prettify.lua (code)
       val = escape(val)
       if spans[t] then
          if t == 'comment' then -- may contain @{ref}
-            val = markup.resolve_inline_references(val)
+            val = prettify.resolve_inline_references(val)
          end
          res:append(span(t,val))
       else
@@ -57,16 +51,3 @@ end
 
 return prettify
 
---[[
-         if t == 'iden' then
-            local tn,vn = tnext(tok)
-            if tn == '.' then
-
-            else
-               res:append(tn)
-               res:append(val)
-            end
-         else
-            res:append(val)
-         end
-]]

ldoc/tools.lua

@@ -342,7 +342,6 @@ end
 
 function M.process_file_list (list, mask, operation, ...)
    local exclude_list = list.exclude and M.files_from_list(list.exclude, mask)
-   if exclude_list then pretty.dump(exclude_list) end
    local function process (f,...)
       f = path.normcase(f)
       f = path.abspath(f)