From 70e1f229098e82b88181d18b7958011dec45605d Mon Sep 17 00:00:00 2001 From: steve donovan Date: Tue, 6 Dec 2011 19:19:09 +0200 Subject: [PATCH] tparam/treturn aliases for type modifiers: display of types with standard template --- docs/doc.md | 49 +++++++-- ldoc.lua | 17 +++- ldoc/doc.lua | 226 ++++++++++++++++++++++++++++++----------- ldoc/html.lua | 15 ++- ldoc/html/ldoc_ltp.lua | 15 ++- ldoc/lang.lua | 50 +++++++-- ldoc/parse.lua | 52 ++++++---- ldoc/tools.lua | 13 ++- tests/example/mylib.c | 10 +- 9 files changed, 340 insertions(+), 107 deletions(-) diff --git a/docs/doc.md b/docs/doc.md index efccbc7..9693e2f 100644 --- a/docs/doc.md +++ b/docs/doc.md @@ -444,12 +444,6 @@ There is an option to simply dump the results of parsing modules. Consider the C This is useful to quickly check for problems; here we see that `createable` did not have a return tag. -There is a more customizable way to process the data, using the `--filter` parameter. This is understood to be a fully qualified function (module + name). For example, try - - $ ldoc --filter pl.pretty.dump mylib.c - -to see a raw dump of the data. (Simply using `dump` as the value here would be a shorthand for `pl.pretty.dump`.) This is potentially very powerful, since you may write arbitrary Lua code to extract the information you need from your project. - LDoc takes this idea of data dumping one step further. If used with the `-m` flag it will look up an installed Lua module and parse it. If it has been marked up in LuaDoc-style then you will get a handy summary of the contents: @plain @@ -626,3 +620,46 @@ This is then styled with `ldoc.css`. Currently the template and stylesheet is ve You may customize how you generate your documentation by specifying an alternative style sheet and/or template, which can be deployed with your project. The parameters are `--style` and `--template`, which give the directories where `ldoc.css` and `ldoc.ltp` are to be found. If `config.ld` contains these variables, they are interpreted slightly differently; if they are true, then it means 'use the same directory as config.ld'; otherwise they must be a valid directory relative to the ldoc invocation. An example of fully customized documentation is `tests/example/style': this is what you could call 'minimal Markdown style' where there is no attempt to tag things (except emphasizing parameter names). The narrative ought to be sufficient, if it is written appropriately. Of course, there's no reason why LDoc must always generate HTML. `--ext` defines what output extension to use; this can also be set in the configuration file. So it's possible to write a template that converts LDoc output to LaTex, for instance. + +## Internal Data Representation + +The `--dump` flag gives a rough text output on the console. But there is a more customizeable way to process the output data generated by LDoc, using the `--filter` parameter. This is understood to be a fully qualified function (module + name). For example, try + + $ ldoc --filter pl.pretty.dump mylib.c + +to see a raw dump of the data. (Simply using `dump` as the value here would be a shorthand for `pl.pretty.dump`.) This is potentially very powerful, since you may write arbitrary Lua code to extract the information you need from your project. + +For instance, a file `custom.lua` like this: + + return { + filter = function (t) + for _, mod in ipairs(t) do + print(mod.type,mod.name,mod.summary) + end + end + } + +Can be used like so: + + ~/LDoc/tests/example$ ldoc --filter custom.filter mylib.c + module mylib A sample C extension. + +The basic data structure is straightforward: it is an array of 'modules' (project-level entities, including scripts) which each contain an `item` array (functions, tables and so forth). + +For instance, to find all functions which don't have a @return tag: + + return { + filter = function (t) + for _, mod in ipairs(t) do + for _, item in ipairs(mod.items) do + if item.type == 'function' and not item.ret then + print(mod.name,item.name,mod.file,item.lineno) + end + end + end + end + } + +The internal naming is not always so consistent; `ret` corresponds to @return, and `params` corresponds to @param. `item.params` is an array of the function parameters, in order; it is also a map from these names to the individual descriptions of the parameters. + +`item.modifiers` is a table where the keys are the tags and the values are arrays of modifier tables. The standard tag aliases `tparam` and `treturn` attach a `type` modifier to their tags. So diff --git a/ldoc.lua b/ldoc.lua index a9a2572..05c1538 100644 --- a/ldoc.lua +++ b/ldoc.lua @@ -112,6 +112,12 @@ function ldoc.alias (a,tag) doc.add_alias(a,tag) end +-- standard aliases -- + +ldoc.alias('tparam',{'param',modifiers={type="$1"}}) +ldoc.alias('treturn',{'return',modifiers={type="$1"}}) +ldoc.alias('tfield',{'field',modifiers={type="$1"}}) + function ldoc.add_language_extension(ext,lang) lang = (lang=='c' and cc) or (lang=='lua' and lua) or quit('unknown language') if ext:sub(1,1) ~= '.' then ext = '.'..ext end @@ -152,7 +158,9 @@ local function read_ldoc_config (fname) directory = '.' end local err - print('reading configuration from '..fname) + if args.filter == 'none' then + print('reading configuration from '..fname) + end local txt,not_found = utils.readfile(fname) if txt then -- Penlight defines loadin for Lua 5.1 as well @@ -280,7 +288,12 @@ local function process_file (f, flist) if ftype then if args.verbose then print(path.basename(f)) end local F,err = parse.file(f,ftype,args) - if err then quit(err) end + if err then + if F then + F:warning("internal LDoc error") + end + quit(err) + end flist:append(F) end end diff --git a/ldoc/doc.lua b/ldoc/doc.lua index 3740810..a7cefb6 100644 --- a/ldoc/doc.lua +++ b/ldoc/doc.lua @@ -18,11 +18,11 @@ local known_tags = { param = 'M', see = 'M', usage = 'M', ['return'] = 'M', field = 'M', author='M'; class = 'id', name = 'id', pragma = 'id', alias = 'id'; copyright = 'S', summary = 'S', description = 'S', release = 'S', license = 'S', - fixme = 'S', todo = 'S', warning = 'S'; + fixme = 'S', todo = 'S', warning = 'S', raise = 'S'; module = 'T', script = 'T', example = 'T', topic = 'T', -- project-level ['function'] = 'T', lfunction = 'T', table = 'T', section = 'T', type = 'T', annotation = 'T'; -- module-level - ['local'] = 'N'; + ['local'] = 'N', export = 'N'; } known_tags._alias = {} known_tags._project_level = { @@ -112,6 +112,17 @@ function File:new_item(tags,line) return item end +function File:export_item (name) + for item in self.items:iter() do + local tags = item.tags + if tags.name == name then + if tags['local'] then + tags['local'] = false + end + end + end +end + function File:finish() local this_mod local items = self.items @@ -187,7 +198,6 @@ function File:finish() these_items.by_name[item.name] = item these_items:append(item) - -- register this item with the iterator this_mod.kinds:add(item,these_items,section_description) else @@ -226,55 +236,116 @@ function Item:_init(tags,file,line) self.formal_args = tags.formal_args tags.formal_args = nil for tag,value in pairs(tags) do - local ttype = known_tags[tag] - if ttype == TAG_MULTI then - if type(value) == 'string' then - value = List{value} - end - self.tags[tag] = value - elseif ttype == TAG_ID then - if type(value) ~= 'string' then - -- such tags are _not_ multiple, e.g. name - self:error("'"..tag.."' cannot have multiple values") - else - self.tags[tag] = tools.extract_identifier(value) - end - elseif ttype == TAG_SINGLE then - self.tags[tag] = value - elseif ttype == TAG_FLAG then - self.tags[tag] = true - else - self:warning ("unknown tag: '"..tag.."' "..tostring(ttype)) + self:set_tag(tag,value) + end +end + +function Item:set_tag (tag,value) + local ttype = known_tags[tag] + if ttype == TAG_MULTI then + if getmetatable(value) ~= List then + value = List{value} end + self.tags[tag] = value + elseif ttype == TAG_ID then + if type(value) ~= 'string' then + -- such tags are _not_ multiple, e.g. name + self:error("'"..tag.."' cannot have multiple values") + else + self.tags[tag] = tools.extract_identifier(value) + end + elseif ttype == TAG_SINGLE then + self.tags[tag] = value + elseif ttype == TAG_FLAG then + self.tags[tag] = true + else + self:warning ("unknown tag: '"..tag.."' "..tostring(ttype)) end end -- preliminary processing of tags. We check for any aliases, and for tags -- which represent types. This implements the shortcut notation. -function Item.check_tag(tags,tag) - tag = doc.get_alias(tag) or tag +function Item.check_tag(tags,tag, value, modifiers) + local alias = doc.get_alias(tag) + if alias then + if type(alias) == 'string' then + tag = alias + else + local avalue,amod + tag, avalue, amod = alias[1],alias.value,alias.modifiers + if avalue then value = avalue..' '..value end + if amod then + modifiers = modifiers or {} + for m,v in pairs(amod) do + local idx = v:match('^%$(%d+)') + if idx then + v, value = value:match('(%S+)%s+(.+)') + end + modifiers[m] = v + end + end + end + end local ttype = known_tags[tag] if ttype == TAG_TYPE then tags.class = tag tag = 'name' end - return tag + return tag, value, modifiers +end + +-- any tag (except name and classs) may have associated modifiers, +-- in the form @tag[m1,...] where m1 is either name1=value1 or name1. +-- At this stage, these are encoded +-- in the tag value table and need to be extracted. + +local function extract_value_modifier (p) + if type(p)=='string' then + return p, { } + elseif type(p) == 'table' then + return p[1], p.modifiers or { } + else + return 'que?',{} + end +end + +local function extract_tag_modifiers (tags) + local modifiers = {} + for tag, value in pairs(tags) do + if type(value)=='table' and value.append then + local tmods = {} + for i, v in ipairs(value) do + v, mods = extract_value_modifier(v) + tmods[i] = mods + value[i] = v + end + modifiers[tag] = tmods + else + value, mods = extract_value_modifier(value) + modifiers[tag] = mods + tags[tag] = value + end + end + return modifiers +end + +local function read_del (tags,name) + local ret = tags[name] + tags[name] = nil + return ret end function Item:finish() local tags = self.tags - self.name = tags.name - self.type = tags.class - self.usage = tags.usage - tags.name = nil - tags.class = nil - tags.usage = nil + self.name = read_del(tags,'name') + self.type = read_del(tags,'class') + self.modifiers = extract_tag_modifiers(tags) + self.usage = read_del(tags,'usage') -- see tags are multiple, but they may also be comma-separated if tags.see then - tags.see = tools.expand_comma_list(tags.see) + tags.see = tools.expand_comma_list(read_del(tags,'see')) end - --if self.type ~= 'function' then print(self.name,self.type) end if doc.project_level(self.type) then -- we are a module, so become one! self.items = List() @@ -282,62 +353,92 @@ function Item:finish() setmetatable(self,Module) elseif not doc.section_tag(self.type) then -- params are either a function's arguments, or a table's fields, etc. - local params if self.type == 'function' then - params = tags.param or List() - if tags['return'] then - self.ret = tags['return'] + self.parameter = 'param' + self.ret = read_del(tags,'return') + self.raise = read_del(tags,'raise') + if tags['local'] then + self.type = 'lfunction' end else - params = tags.field or List() + self.parameter = 'field' end - tags.param = nil + local params = read_del(tags,self.parameter) local names, comments, modifiers = List(), List(), List() - for p in params:iter() do - local line, mods - if type(p)=='string' then line, mods = p, { } - else line, mods = p[1], p.modifiers or { } end - modifiers:append(mods) - local name, comment = line :match('%s*([%w_%.:]+)(.*)') - assert(name, "bad param name format") - names:append(name) - comments:append(comment) + if params then + for line in params:iter() do + local name, comment = line :match('%s*([%w_%.:]+)(.*)') + assert(name, "bad param name format") + names:append(name) + comments:append(comment) + end end -- not all arguments may be commented -- if self.formal_args then -- however, ldoc allows comments in the arg list to be used local fargs = self.formal_args - for a in fargs:iter() do - if not names:index(a) then - names:append(a) - comments:append (fargs.comments[a] or '') + for i,name in ipairs(fargs) do + if params then -- explicit set of param tags + if names[i] ~= name then + self:warning("param and formal argument name mismatch: '"..name.."' '"..tostring(names[i]).."'") + end + else + names:append(name) + local comment = fargs.comments[name] + comments:append (comment or '') end end end + + -- the comments are associated with each parameter by + -- adding name-value pairs to the params list (this is + -- also done for any associated modifiers) self.params = names - self.modifiers = modifiers + local pmods = self.modifiers[self.parameter] for i,name in ipairs(self.params) do self.params[name] = comments[i] + if pmods then + pmods[name] = pmods[i] + end end + + -- build up the string representation of the argument list, + -- using any opt and optchain modifiers if present. + -- For instance, '(a [, b])' if b is marked as optional + -- with @param[opt] b local buffer, npending = { }, 0 local function acc(x) table.insert(buffer, x) end for i = 1, #names do - local m = modifiers[i] - if m then + local m = pmods and pmods[i] + if m then if not m.optchain then - acc ((']'):rep(npending)) - npending=0 + acc ((']'):rep(npending)) + npending=0 end if m.opt or m.optchain then acc('['); npending=npending+1 end - end - if i>1 then acc (', ') end - acc(names[i]) + end + if i>1 then acc (', ') end + acc(names[i]) end acc ((']'):rep(npending)) self.args = '('..table.concat(buffer)..')' end end +function Item:type_of_param(p) + local mods = self.modifiers[self.parameter] + if not mods then return '' end + local mparam = mods[p] + return mparam and mparam.type or '' +end + +function Item:type_of_ret(idx) + local rparam = self.modifiers['return'][idx] + return rparam and rparam.type or '' +end + + + function Item:warning(msg) local name = self.file and self.file.filename if type(name) == 'table' then pretty.dump(name); name = '?' end @@ -376,6 +477,9 @@ end function Module:process_see_reference (s,modules) local mod_ref,fun_ref,name,packmod + if not s:match '^[%w_%.%:]+$' or not s:match '[%w_]$' then + return nil, "malformed see reference: '"..s..'"' + end -- is this a fully qualified module name? local mod_ref = modules.by_name[s] if mod_ref then return reference(s, mod_ref,nil) end @@ -513,7 +617,7 @@ function Item:dump(verbose) end function doc.filter_objects_through_function(filter, module_list) - local quit = utils.quit + local quit, quote = utils.quit, tools.quote if filter == 'dump' then filter = 'pl.pretty.dump' end local mod,name = tools.split_dotted_name(filter) local ok,P = pcall(require,mod) diff --git a/ldoc/html.lua b/ldoc/html.lua index 2f52f59..4dbeaf0 100644 --- a/ldoc/html.lua +++ b/ldoc/html.lua @@ -15,7 +15,7 @@ local template = require 'pl.template' local tools = require 'ldoc.tools' - +local markup = require 'ldoc.markup' local html = {} @@ -84,6 +84,18 @@ function html.generate_output(ldoc, args, project) end)) end + function ldoc.typename (tp) + if not tp then return '' end + return (tp:gsub('%a[%w_%.]*',function(name) + local ref,err = markup.process_reference(name) + if ref then + return ('%s '):format(ldoc.href(ref),name) + else + return ''..name..' ' + end + end)) + end + local module_template,err = utils.readfile (path.join(args.template,ldoc.templ)) if not module_template then quit("template not found at '"..args.template.."' Use -l to specify directory containing ldoc.ltp") @@ -91,6 +103,7 @@ function html.generate_output(ldoc, args, project) local css = ldoc.css ldoc.output = args.output + ldoc.ipairs = ipairs -- in single mode there is one module and the 'index' is the -- documentation for that module. diff --git a/ldoc/html/ldoc_ltp.lua b/ldoc/html/ldoc_ltp.lua index d5b0eb6..a0f7661 100644 --- a/ldoc/html/ldoc_ltp.lua +++ b/ldoc/html/ldoc_ltp.lua @@ -134,13 +134,14 @@ return [==[ $(display_name(item))
- $(M(item.summary..' '..(item.description or ''),item)) + $(M((item.summary or '?')..' '..(item.description or ''),item)) # if show_parms and item.params and #item.params > 0 then

$(module.kinds:type_of(item).subnames):

# end -- if params @@ -159,12 +160,18 @@ return [==[ # local li,il = use_li(item.ret)

Returns:

    -# for r in iter(item.ret) do - $(li)$(M(r,item))$(il) +# for i,r in ldoc.ipairs(item.ret) do +# local tp = ldoc.typename(item:type_of_ret(i)) + $(li)$(tp)$(M(r,item))$(il) # end -- for
# end -- if returns +# if show_return and item.raise then +

Raises:

+ $(M(item.raise,item)) +# end + # if item.see then # local li,il = use_li(item.see)

see also:

diff --git a/ldoc/lang.lua b/ldoc/lang.lua index a06c8c6..376a7f2 100644 --- a/ldoc/lang.lua +++ b/ldoc/lang.lua @@ -55,8 +55,12 @@ end function Lang:parse_extra (tags,tok) end -function Lang:parse_usage (tags, tok) - return nil, "@usage deduction not implemented for this language" +function Lang:is_module_modifier () + return false +end + +function Lang:parse_module_modifier (tags, tok) + return nil, "@usage or @exports deduction not implemented for this language" end @@ -160,6 +164,15 @@ function Lua:item_follows(t,v,tok) tags.name = name end end + elseif t == 'keyword' and v == 'return' then -- case [5] + case = 5 + if tnext(tok) ~= '{' then + return nil + end + parser = function(tags,tok) + tags.class = 'table' + parse_lua_table(tags,tok) + end end return parser, is_local, case end @@ -175,13 +188,32 @@ function Lua:parse_extra (tags,tok,case) end end -function Lua:parse_usage (tags, tok) - if tags.class ~= 'field' then return nil,"cannot deduce @usage" end - local t1= tnext(tok) - local t2 = tok() - if t1 ~= '[' or t1 ~= '[' then return nil, 'not a long string' end - t, v = tools.grab_block_comment('',tok,'%]%]') - return true, v +-- For Lua, a --- @usage comment means that a long +-- string containing the usage follows, which we +-- use to update the module usage tag. Likewise, the @export +-- tag alone in a doc comment refers to the following returned +-- Lua table of functions + + +function Lua:is_module_modifier (tags) + return tags.summary == '' and (tags.usage or tags.export) +end + +function Lua:parse_module_modifier (tags, tok, F) + if tags.usage then + if tags.class ~= 'field' then return nil,"cannot deduce @usage" end + local t1= tnext(tok) + local t2 = tok() + if t1 ~= '[' or t2 ~= '[' then return nil, 'not a long string' end + t, v = tools.grab_block_comment('',tok,'%]%]') + return true, v, 'usage' + elseif tags.export then + if tags.class ~= 'table' then return nil, "cannot deduce @export" end + for f in tags.formal_args:iter() do + F:export_item(f) + end + return true + end end diff --git a/ldoc/parse.lua b/ldoc/parse.lua index ea34ae9..774b8e8 100644 --- a/ldoc/parse.lua +++ b/ldoc/parse.lua @@ -66,16 +66,16 @@ local function extract_tags (s) end -- and strip(description) ? local tags = {summary=summary and strip(summary) or '',description=description or ''} for _,item in ipairs(tag_items) do - local tag, value, modifiers = unpack(item) - tag = Item.check_tag(tags,tag) + local tag, value, modifiers = Item.check_tag(tags,unpack(item)) value = strip(value) + if modifiers then value = { value, modifiers=modifiers } end local old_value = tags[tag] if not old_value then -- first element tags[tag] = value - elseif type(old_value)=='table' and old_value.append then -- append to existing list - old_value :append (value) + elseif type(old_value)=='table' and old_value.append then -- append to existing list + old_value :append (value) else -- upgrade string->list tags[tag] = List{old_value, value} end @@ -83,6 +83,11 @@ local function extract_tags (s) return Map(tags) end +local _xpcall = xpcall +if true then + _xpcall = function(f) return true, f() end +end + -- parses a Lua or C file, looking for ldoc comments. These are like LuaDoc comments; @@ -96,7 +101,7 @@ local function parse_file(fname,lang, package) local line,f = 1 local F = File(fname) local module_found, first_comment = false,true - local current_item + local current_item, module_item local tok,f = lang.lexer(fname) @@ -122,6 +127,7 @@ local function parse_file(fname,lang, package) tags.class = 'module' local item = F:new_item(tags,lineno()) item.old_style = old_style + module_item = item end local mod @@ -141,6 +147,7 @@ local function parse_file(fname,lang, package) end end end + local ok, err = xpcall(function() while t do if t == 'comment' then local comment = {} @@ -163,7 +170,7 @@ local function parse_file(fname,lang, package) end end - if not t then break end -- no more file! +-- if not t then break end -- no more file! if t == 'space' then t,v = tnext(tok) end @@ -189,16 +196,23 @@ local function parse_file(fname,lang, package) -- if the item has an explicit name or defined meaning -- then don't continue to do any code analysis! if tags.name then + if not tags.class then + F:warning("no type specified, assuming function: '"..tags.name.."'") + tags.class = 'function' + end item_follows, is_local = false, false - elseif tags.summary == '' and tags.usage then - -- For Lua, a --- @usage comment means that a long - -- string containing the usage follows, which we - -- use to update the module usage tag + elseif lang:is_module_modifier (tags) then + if not item_follows then + F:warning("@usage or @export followed by unknown code") + break + end item_follows(tags,tok) - local res, value = lang:parse_usage(tags,tok) + local res, value, tagname = lang:parse_module_modifier(tags,tok,F) if not res then F:warning(fname,value,1); break else - current_item.tags.usage = {value} + if tagname then + module_item.set_tag(tagname,value) + end -- don't continue to make an item! ldoc_comment = false end @@ -227,7 +241,7 @@ local function parse_file(fname,lang, package) -- end of a block of document comments if ldoc_comment and tags then - local line = t ~= nil and lineno() or 666 + local line = t ~= nil and lineno() if t ~= nil then if item_follows then -- parse the item definition item_follows(tags,tok) @@ -235,9 +249,8 @@ local function parse_file(fname,lang, package) lang:parse_extra(tags,tok,case) end end - -- local functions treated specially - if tags.class == 'function' and (is_local or tags['local']) then - tags.class = 'lfunction' + if is_local or tags['local'] then + tags['local'] = true end if tags.name then current_item = F:new_item(tags,line) @@ -248,14 +261,17 @@ local function parse_file(fname,lang, package) end if t ~= 'comment' then t,v = tok() end end + end,debug.traceback) + if not ok then return F, err end if f then f:close() end return F end function parse.file(name,lang, args) local F,err = parse_file(name,lang, args.package) - if err then return nil,err end - F:finish() + if err then return F,err end + local ok,err = xpcall(function() F:finish() end,debug.traceback) + if not ok then return F,err end return F end diff --git a/ldoc/tools.lua b/ldoc/tools.lua index 808965c..5ade1e2 100644 --- a/ldoc/tools.lua +++ b/ldoc/tools.lua @@ -269,7 +269,9 @@ function M.get_parameters (tok,endtoken,delim) end for i = 1,#ltl do + --print('check',i,ltl[i],#ltl[i]) local tl = ltl[i] + if #tl > 0 then if type_of(tl[1]) == 'comment' then if i > 1 then set_comment(i-1,tl[1]) end if #tl > 1 then @@ -284,6 +286,7 @@ function M.get_parameters (tok,endtoken,delim) set_comment(i,last_tok) end end + end end return args @@ -340,9 +343,17 @@ function M.grab_block_comment (v,tok,patt) return 'comment',res end +local prel = path.normcase('/[^/]-/%.%.') + function M.abspath (f) - return path.normcase(path.abspath(f)) + local count + local res = path.normcase(path.abspath(f)) + while true do + res,count = res:gsub(prel,'') + if count == 0 then break end + end + return res end function M.process_file_list (list, mask, operation, ...) diff --git a/tests/example/mylib.c b/tests/example/mylib.c index 441a2c2..fd9c675 100644 --- a/tests/example/mylib.c +++ b/tests/example/mylib.c @@ -25,11 +25,11 @@ static int l_createtable (lua_State *L) { /*** Solve a quadratic equation. @function solve -@param a coefficient of x^2 -@param b coefficient of x -@param c constant -@return first root -@return second root +@tparam num a coefficient of x^2 +@tparam num b coefficient of x +@tparam num c constant +@treturn num first root +@treturn num second root */ static int l_solve (lua_State *L) { double a = lua_tonumber(L,1); // coeff of x*x