From 788d8f24bd5bf540dd2130c3eaa477e38ef5630b Mon Sep 17 00:00:00 2001
From: steve donovan <steve.j.donovan@gmail.com>
Date: Mon, 11 Jul 2011 15:40:44 +0200
Subject: [PATCH] Topics working with section references; cross-references
 starting to work

---
 html/ldoc.ltp     |   2 +-
 ldoc.lua          | 134 +++++++++++++++++++---------------------------
 ldoc/doc.lua      |  55 ++++++++++++++-----
 ldoc/markup.lua   |  89 ++++++++++++++++++++++++------
 ldoc/prettify.lua |  24 +++++++--
 ldoc/tools.lua    |  33 ++++++++++++
 6 files changed, 224 insertions(+), 113 deletions(-)

diff --git a/html/ldoc.ltp b/html/ldoc.ltp
index ce7cba3..e25c2a3 100644
--- a/html/ldoc.ltp
+++ b/html/ldoc.ltp
@@ -67,7 +67,7 @@
 #  if mod.name == this_mod then -- highlight current module, link to others
   <li><strong>$(mod.name)</strong></li>
 #  else
-  <li><a href="$(ldoc.ref_to_module(mod,module,kind)).html">$(mod.name)</a></li>
+  <li><a href="$(ldoc.ref_to_module(mod))">$(mod.name)</a></li>
 #  end
 # end
 #end
diff --git a/ldoc.lua b/ldoc.lua
index d75ea3e..ad8de17 100644
--- a/ldoc.lua
+++ b/ldoc.lua
@@ -67,6 +67,7 @@ end
 
 ProjectMap:add_kind('module','Modules')
 ProjectMap:add_kind('script','Scripts')
+ProjectMap:add_kind('topic','Topics')
 ProjectMap:add_kind('example','Examples')
 
 
@@ -416,6 +417,11 @@ function add_language_extension (ext,lang)
    file_types[ext] = lang
 end
 
+--------- processing files ---------------------
+-- ldoc may be given a file, or a directory. `args.file` may also be specified in config.ld
+-- where it is a list of files or directories. If specified on the command-line, we have
+-- to find an optional associated config.ld, if not already loaded.
+
 local function process_file (f, file_list)
    local ext = path.extension(f)
    local ftype = file_types[ext]
@@ -426,40 +432,7 @@ local function process_file (f, file_list)
    end
 end
 
-local process_file_list, files_from_list
-
-function process_file_list (list, mask, operation, ...)
-   local exclude_list = list.exclude and files_from_list(list.exclude, mask)
-   if exclude_list then pretty.dump(exclude_list) end
-   local function process (f,...)
-      f = path.normcase(f)
-      f = path.abspath(f)
-      if exclude_list and exclude_list:index(f) == nil then
-         operation(f, ...)
-      end
-   end
-   for _,f in ipairs(list) do
-      if path.isdir(f) then
-         local files = List(dir.getallfiles(f,mask))
-         for f in files:iter() do
-            process(f,...)
-         end
-      elseif path.isfile(f) then
-         process(f,...)
-      else
-         quit("file or directory does not exist: "..quote(f))
-      end
-   end
-end
-
-function files_from_list (list, mask)
-   local excl = List()
-   process_file_list (list, mask, function(f)
-      excl:append(f)
-   end)
-   return excl
-end
-
+local process_file_list = tools.process_file_list
 
 if type(args.file) == 'table' then
    -- this can only be set from config file so we can assume it's already read
@@ -503,27 +476,53 @@ end
 
 setup_package_base()
 
+
 local multiple_files = #file_list > 1
 local first_module
 
+------ 'Special' Project-level entities ---------------------------------------
+-- Examples and Topics do not contain code to be processed for doc comments.
+-- Instead, they are intended to be rendered nicely as-is, whether as pretty-lua
+-- or as Markdown text. Treating them as 'modules' does stretch the meaning of
+-- of the term, but allows them to be treated much as modules or scripts.
+-- They define an item 'body' field (containing the file's text) and a 'postprocess'
+-- field which is used later to convert them into HTML. They may contain @{ref}s.
+
+local function add_special_project_entity (f,tags,process)
+   local F = File(f)
+   tags.name = path.basename(f)
+   local text = utils.readfile(f)
+   local item = F:new_item(tags,1)
+   if process then
+      text = process(F, text)
+   end
+   F:finish()
+   file_list:append(F)
+   item.body = text
+   return item
+end
+
 if type(ldoc.examples) == 'table' then
    local prettify = require 'ldoc.prettify'
 
    local function process_example (f, file_list)
-      local F = File(f)
-      local tags = {
-         name = path.basename(f),
+      local item = add_special_project_entity(f,{
          class = 'example',
-      }
-      local item = F:new_item(tags,1)
-      F:finish()
-      item.body = prettify.lua(f)
-      file_list:append(F)
+      })
+      item.postprocess = prettify.lua
    end
 
    process_file_list (ldoc.examples, '*.lua', process_example, file_list)
 end
 
+if type(ldoc.readme) == 'string' then
+   local item = add_special_project_entity(ldoc.readme,{
+      class = 'topic'
+   }, markup.add_sections)
+   item.postprocess = markup.create(ldoc, 'markdown')
+end
+
+---- extract modules from the file objects, resolve references and sort appropriately ---
 
 local project = ProjectMap()
 
@@ -551,12 +550,14 @@ table.sort(module_list,function(m1,m2)
    return m1.name < m2.name
 end)
 
+-------- three ways to dump the object graph after processing -----
+
 -- ldoc -m will give a quick & dirty dump of the module's documentation;
 -- using -v will make it more verbose
 if args.module then
    if #module_list == 0 then quit("no modules found") end
    if args.module == true then
-      F:dump(args.verbose)
+      file_list[1]:dump(args.verbose)
    else
       local fun = module_list[1].items.by_name[args.module]
       if not fun then quit(quote(args.module).." is not part of "..quote(args.file)) end
@@ -565,6 +566,7 @@ if args.module then
    return
 end
 
+-- ldoc --dump will do the same as -m, except for the currently specified files
 if args.dump then
    for mod in module_list:iter() do
       mod:dump(true)
@@ -572,31 +574,10 @@ if args.dump then
    os.exit()
 end
 
-
+-- ldoc --filter mod.name will load the module `mod` and pass the object graph
+-- to the function `name`. As a special case --filter dump will use pl.pretty.dump.
 if args.filter ~= 'none' then
-   local mod,name = tools.split_dotted_name(args.filter)
-   local ok,P = pcall(require,mod)
-   if not ok then quit("cannot find module "..quote(mod)) end
-   local ok,f = pcall(function() return P[name] end)
-   if not ok or type(f) ~= 'function' then quit("dump module: no function "..quote(name)) end
-
-   -- clean up some redundant and cyclical references--
-   module_list.by_name = nil
-   for mod in module_list:iter() do
-      mod.kinds = nil
-      mod.file = mod.file.filename
-      for item in mod.items:iter() do
-         item.module = nil
-         item.file = nil
-         item.formal_args = nil
-         item.tags['return'] = nil
-         item.see = nil
-      end
-      mod.items.by_name = nil
-   end
-
-   local ok,err = pcall(f,module_list)
-   if not ok then quit("dump failed: "..err) end
+   doc.filter_objects_through_function(args.filter, module_list)
    os.exit()
 end
 
@@ -667,10 +648,7 @@ function ldoc.href(see)
    if see.href then -- explict reference, e.g. to Lua manual
       return see.href
    else
-      -- usually the module name, except when we have a single module
-      local doc_name = see.mod
-      if ldoc.single then doc_name = args.output end
-      return doc_name..'.html#'..see.name
+      return ldoc.ref_to_module(see.mod)..'#'..see.name
    end
 end
 
@@ -679,11 +657,10 @@ end
 -- then linking to another kind is `../kind/name`; to the same kind is just `name`.
 -- If we are in the root, then it is `kind/name`.
 
-function ldoc.ref_to_module (mod,module,kind)
+function ldoc.ref_to_module (mod)
    local base = "" -- default: same directory
+   local kind, module = mod.kind, ldoc.module
    local name = mod.name -- default: name of module
-   local single_mod = ldoc.single and ldoc.root
-   kind = kind:lower()
    if not ldoc.single then
       if module then -- we are in kind/
          if module.type ~= type then -- cross ref to ../kind/
@@ -693,7 +670,6 @@ function ldoc.ref_to_module (mod,module,kind)
          base = kind..'/'
       end
    else -- single module
-      --print('mod',mod.name,mod.type,module.type,first_module.type)
       if mod == first_module then
          name = ldoc.output
          if not ldoc.root then base = '../' end
@@ -705,14 +681,13 @@ function ldoc.ref_to_module (mod,module,kind)
          end
       end
    end
-   --print('res',base..name)
-   return base..name
+   return base..name..'.html'
 end
 
 
 local function generate_output()
    local check_directory, check_file, writefile = tools.check_directory, tools.check_file, tools.writefile
-   ldoc.single = not multiple_files
+   ldoc.single = not multiple_files and first_module or nil
    ldoc.log = print
    ldoc.kinds = project
    ldoc.css = css
@@ -752,6 +727,9 @@ local function generate_output()
          for m in modules() do
             ldoc.module = m
             ldoc.body = m.body
+            if ldoc.body then
+               ldoc.body = m.postprocess(ldoc.body)
+            end
             out,err = template.substitute(module_template,{
                module=m,
                ldoc = ldoc
diff --git a/ldoc/doc.lua b/ldoc/doc.lua
index a12f75f..333e889 100644
--- a/ldoc/doc.lua
+++ b/ldoc/doc.lua
@@ -18,15 +18,16 @@ local known_tags = {
    param = 'M', see = 'M', usage = 'M', ['return'] = 'M', field = 'M', author='M';
    class = 'id', name = 'id', pragma = 'id', alias = 'id';
    copyright = 'S', summary = 'S', description = 'S', release = 'S', license = 'S';
-   module = 'T', script = 'T', example = 'T',
-   ['function'] = 'T', lfunction = 'T', table = 'T', section = 'T', type = 'T';
+   module = 'T', script = 'T', example = 'T', topic = 'T', -- project-level
+   ['function'] = 'T', lfunction = 'T', table = 'T', section = 'T', type = 'T'; -- module-level
    ['local'] = 'N';
 }
 known_tags._alias = {}
 known_tags._project_level = {
    module = true,
    script = true,
-   example = true
+   example = true,
+   topic = true
 }
 
 local TAG_MULTI,TAG_ID,TAG_SINGLE,TAG_TYPE,TAG_FLAG = 'M','id','S','T','N'
@@ -82,7 +83,7 @@ function File:_init(filename)
 end
 
 function File:new_item(tags,line)
-   local item = Item(tags,self,line)
+   local item = Item(tags,self,line or 1)
    self.items:append(item)
    return item
 end
@@ -98,12 +99,10 @@ function File:finish()
          if item.type == 'module' then
             -- if name is 'package.mod', then mod_name is 'mod'
             package,mname = split_dotted_name(this_mod.name)
-            if not package then
-               mname = this_mod.name
-               package = ''
-            else
-               package = package
-            end
+         end
+         if not package then
+            mname = this_mod.name
+            package = ''
          end
          self.modules:append(this_mod)
          this_mod.package = package
@@ -203,7 +202,7 @@ function Item:_init(tags,file,line)
       elseif ttype == TAG_FLAG then
          self.tags[tag] = true
       else
-         self:warning ("unknown tag: '"..tag.."'")
+         self:warning ("unknown tag: '"..tag.."' "..tostring(ttype))
       end
    end
 end
@@ -291,7 +290,7 @@ end
 
 function Module:hunt_for_reference (packmod, modules)
    local mod_ref
-   local package = self.package
+   local package = self.package or ''
    repeat -- same package?
       local nmod = package..'.'..packmod
       mod_ref = modules.by_name[nmod]
@@ -307,7 +306,7 @@ local function reference (s, mod_ref, item_ref)
    if item_ref and item_ref.type == 'type' then
       name = 'Class_'..name
    end
-   return {mod = mod_ref.name, name = name, label=s}
+   return {mod = mod_ref, name = name, label=s}
 end
 
 function Module:process_see_reference (s,modules)
@@ -391,6 +390,8 @@ function Module:mask_locals ()
    self.kinds['Local Functions'] = nil
 end
 
+--------- dumping out modules and items -------------
+
 function Module:dump(verbose)
    print '----'
    print(self.type..':',self.name,self.summary)
@@ -431,5 +432,33 @@ function Item:dump(verbose)
    end
 end
 
+function doc.filter_objects_through_function(filter, module_list)
+   local quit = utils.quit
+   if filter == 'dump' then filter = 'pl.pretty.dump' end
+   local mod,name = tools.split_dotted_name(filter)
+   local ok,P = pcall(require,mod)
+   if not ok then quit("cannot find module "..quote(mod)) end
+   local ok,f = pcall(function() return P[name] end)
+   if not ok or type(f) ~= 'function' then quit("dump module: no function "..quote(name)) end
+
+   -- clean up some redundant and cyclical references--
+   module_list.by_name = nil
+   for mod in module_list:iter() do
+      mod.kinds = nil
+      mod.file = mod.file.filename
+      for item in mod.items:iter() do
+         item.module = nil
+         item.file = nil
+         item.formal_args = nil
+         item.tags['return'] = nil
+         item.see = nil
+      end
+      mod.items.by_name = nil
+   end
+
+   local ok,err = pcall(f,module_list)
+   if not ok then quit("dump failed: "..err) end
+end
+
 return doc
 
diff --git a/ldoc/markup.lua b/ldoc/markup.lua
index 0497256..5c86f6a 100644
--- a/ldoc/markup.lua
+++ b/ldoc/markup.lua
@@ -4,6 +4,7 @@
 -- be the general module for managing other formats as well.
 
 require 'pl'
+local doc = require 'ldoc.doc'
 local utils = require 'pl.utils'
 local quit = utils.quit
 local markup = {}
@@ -38,34 +39,83 @@ function markup.insert_markdown_lines (txt)
    return table.concat(res,'\n')
 end
 
--- inline <references> use same lookup as @see
-function markup.resolve_inline_references (ldoc, txt)
-   return (txt:gsub('@{([%w_%.]-)}',function(name)
-      local ref,err = ldoc.module:process_see_reference(name,ldoc.modules)
-      if not ref then
-         if ldoc.item then ldoc.item:warning(err)
-         else io.stderr:write(err,'\n')
-         end
-         return ''
+-- for readme text, the idea here is to insert module sections at ## so that
+-- they can appear in the contents list as a ToC
+function markup.add_sections(F, txt)
+   local res, append = {}, table.insert
+   local last_indent, start_indent, skip = -1, -1, false
+   for line in stringx.lines(txt) do
+      local title = line:match '^##[^#]%s*(.+)'
+      if title then
+         -- some serious hackery. We force sections into this 'module',
+         -- and ensure that there is a dummy item so that the section
+         -- is not empty.
+         local section = title:gsub('%A','_')
+         F:new_item {
+            name = section,
+            class = 'section',
+            summary = title
+         }
+         F:new_item {
+            name = 'dumbo',
+            class = 'function',
+         }
+         append(res,('<a id="%s"></a>\n'):format(section))
+         append(res,line)
+      else
+         append(res,line)
       end
-      local label = ref.label:gsub('_','\\_')
-      local res = ('<a href="%s">%s</a>'):format(ldoc.href(ref),label)
-      return res
-   end))
+   end
+   return table.concat(res,'\n')
+end
+
+local function handle_reference (ldoc, name)
+   local ref,err = markup.process_reference(name)
+   if not ref then
+      if ldoc.item then ldoc.item:warning(err)
+      else
+        io.stderr:write(err,'\n')
+      end
+      return ''
+   end
+   local label = ref.label
+   if not ldoc.plain then -- a nastiness with markdown.lua and underscores
+      label = label:gsub('_','\\_')
+   end
+   local res = ('<a href="%s">%s</a>'):format(ldoc.href(ref),label)
+   return res
+end
+
+local ldoc_handle_reference
+
+-- inline <references> use same lookup as @see
+local function resolve_inline_references (ldoc, txt)
+   return (txt:gsub('@{([%w_%.%-]-)}',ldoc_handle_reference))
 end
 
 function markup.create (ldoc, format)
+   local processor
+   ldoc_handle_reference = utils.bind1(handle_reference,ldoc)
+   markup.plain = true
+   markup.process_reference = function(name)
+      local mod = ldoc.single or ldoc.module
+      return mod:process_see_reference(name, ldoc.modules)
+   end
+   markup.href = function(ref)
+      return ldoc.href(ref)
+   end
    if format == 'plain' then
-      return function(txt)
+      processor = function(txt)
          if txt == nil then return '' end
-         return markup.resolve_inline_references(ldoc, txt)
+         return resolve_inline_references(ldoc, txt)
       end
    else
       local ok,formatter = pcall(require,format)
       if not ok then quit("cannot load formatter: "..format) end
-      return function (txt)
+      markup.plain = false
+      processor = function (txt)
          if txt == nil then return '' end
-         txt = markup.resolve_inline_references(ldoc, txt)
+         txt = resolve_inline_references(ldoc, txt)
          if txt:find '\n' and not ldoc.classic_markdown then -- multiline text
             txt = markup.insert_markdown_lines(txt)
          end
@@ -74,6 +124,11 @@ function markup.create (ldoc, format)
          return (txt:gsub('^%s*<p>(.+)</p>%s*$','%1'))
       end
    end
+   markup.resolve_inline_references = function(txt)
+      return resolve_inline_references(ldoc, txt)
+   end
+   markup.processor = processor
+   return processor
 end
 
 
diff --git a/ldoc/prettify.lua b/ldoc/prettify.lua
index 0d8f924..7b488e3 100644
--- a/ldoc/prettify.lua
+++ b/ldoc/prettify.lua
@@ -5,6 +5,8 @@
 -- `@{example:test-fun}`.
 require 'pl'
 local lexer = require 'ldoc.lexer'
+local markup = require 'ldoc.markup'
+local tnext = lexer.skipws
 local prettify = {}
 
 local escaped_chars = {
@@ -29,10 +31,7 @@ end
 
 local spans = {keyword=true,number=true,string=true,comment=true}
 
-function prettify.lua (file)
-   local code,err = utils.readfile(file)
-   if not code then return nil,err end
-
+function prettify.lua (code)
    local res = List()
    res:append(header)
    res:append '<pre>\n'
@@ -43,6 +42,9 @@ function prettify.lua (file)
    while t do
       val = escape(val)
       if spans[t] then
+         if t == 'comment' then -- may contain @{ref}
+            val = markup.resolve_inline_references(val)
+         end
          res:append(span(t,val))
       else
          res:append(val)
@@ -54,3 +56,17 @@ function prettify.lua (file)
 end
 
 return prettify
+
+--[[
+         if t == 'iden' then
+            local tn,vn = tnext(tok)
+            if tn == '.' then
+
+            else
+               res:append(tn)
+               res:append(val)
+            end
+         else
+            res:append(val)
+         end
+]]
diff --git a/ldoc/tools.lua b/ldoc/tools.lua
index e36e215..f4e6fbd 100644
--- a/ldoc/tools.lua
+++ b/ldoc/tools.lua
@@ -84,6 +84,7 @@ function KindMap:add (item,items,description)
       --print(kname,description)
       self.klass.descriptions[kname] = description
    end
+   item.kind = kname:lower()
 end
 
 -- KindMap has a 'class constructor' which is used to modify
@@ -339,6 +340,38 @@ function M.grab_block_comment (v,tok,end1,end2)
    return 'comment',res
 end
 
+function M.process_file_list (list, mask, operation, ...)
+   local exclude_list = list.exclude and M.files_from_list(list.exclude, mask)
+   if exclude_list then pretty.dump(exclude_list) end
+   local function process (f,...)
+      f = path.normcase(f)
+      f = path.abspath(f)
+      if not exclude_list or exclude_list and exclude_list:index(f) == nil then
+         operation(f, ...)
+      end
+   end
+   for _,f in ipairs(list) do
+      if path.isdir(f) then
+         local files = List(dir.getallfiles(f,mask))
+         for f in files:iter() do
+            process(f,...)
+         end
+      elseif path.isfile(f) then
+         process(f,...)
+      else
+         quit("file or directory does not exist: "..quote(f))
+      end
+   end
+end
+
+function M.files_from_list (list, mask)
+   local excl = List()
+   M.process_file_list (list, mask, function(f)
+      excl:append(f)
+   end)
+   return excl
+end
+
 
 
 return tools