Aire-One
/
LDoc2tl
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Several changes; if a single file is given, the module will be written directly to the output directory (no index). --output can change this name (works). Can define sections

This commit is contained in:
steve donovan 2011-06-06 18:38:02 +02:00
parent 74455c089a
commit be9c3f2f70
3 changed files with 765 additions and 688 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

499
doc.lua
View File

@ -15,15 +15,15 @@ local split_dotted_name = tools.split_dotted_name
-- - tags with a single value, like 'release' (TAG_SINGLE)
-- - tags which represent a type, like 'function' (TAG_TYPE)
local known_tags = {
param = 'M', see = 'M', usage = 'M', ['return'] = 'M', field = 'M', author='M';
class = 'id', name = 'id', pragma = 'id', alias = 'id';
copyright = 'S', summary = 'S', description = 'S', release = 'S';
module = 'T', script = 'T',['function'] = 'T', table = 'T'
param = 'M', see = 'M', usage = 'M', ['return'] = 'M', field = 'M', author='M';
class = 'id', name = 'id', pragma = 'id', alias = 'id';
copyright = 'S', summary = 'S', description = 'S', release = 'S', license = 'S';
module = 'T', script = 'T',['function'] = 'T', table = 'T', section = 'T', type = 'T';
}
known_tags._alias = {}
known_tags._project_level = {
module = true,
script = true
module = true,
script = true
}
local TAG_MULTI,TAG_ID,TAG_SINGLE,TAG_TYPE = 'M','id','S','T'
@ -31,25 +31,30 @@ doc.TAG_MULTI,doc.TAG_ID,doc.TAG_SINGLE,doc.TAG_TYPE = TAG_MULTI,TAG_ID,TAG_SING
-- add a new tag.
function doc.add_tag(tag,type,project_level)
if not known_tags[tag] then
known_tags[tag] = type
known_tags._project_level[tag] = project_level
end
if not known_tags[tag] then
known_tags[tag] = type
known_tags._project_level[tag] = project_level
end
end
-- add an alias to an existing tag (exposed through ldoc API)
function doc.add_alias (a,tag)
known_tags._alias[a] = tag
known_tags._alias[a] = tag
end
-- get the tag alias value, if it exists.
function doc.get_alias(tag)
return known_tags._alias[tag]
return known_tags._alias[tag]
end
-- is it a'project level' tag, such as 'module' or 'script'?
function doc.project_level(tag)
return known_tags._project_level[tag]
return known_tags._project_level[tag]
end
-- is it a section tag, like 'type' (class) or 'section'?
function doc.section_tag (tag)
return tag == 'section' or tag == 'type'
end
-- we process each file, resulting in a File object, which has a list of Item objects.
@ -67,172 +72,203 @@ doc.Item = Item
doc.Module = Module
function File:_init(filename)
self.filename = filename
self.items = List()
self.modules = List()
self.filename = filename
self.items = List()
self.modules = List()
end
function File:new_item(tags,line)
local item = Item(tags)
self.items:append(item)
item.file = self
item.lineno = line
return item
local item = Item(tags)
self.items:append(item)
item.file = self
item.lineno = line
return item
end
function File:finish()
local this_mod
local items = self.items
for item in items:iter() do
item:finish()
if doc.project_level(item.type) then
this_mod = item
-- if name is 'package.mod', then mod_name is 'mod'
local package,mname = split_dotted_name(this_mod.name)
if not package then
mname = this_mod.name
package = ''
local this_mod
local items = self.items
for item in items:iter() do
item:finish()
if doc.project_level(item.type) then
this_mod = item
-- if name is 'package.mod', then mod_name is 'mod'
local package,mname = split_dotted_name(this_mod.name)
if not package then
mname = this_mod.name
package = ''
else
package = package .. '.'
end
self.modules:append(this_mod)
this_mod.package = package
this_mod.mod_name = mname
this_mod.kinds = ModuleMap() -- the iterator over the module contents
elseif doc.section_tag(item.type) then
local display_name = item.name
if display_name == 'end' then
this_mod.section = nil
else
local summary = item.summary:gsub('%.$','')
if item.type == 'type' then
display_name = 'Class '..item.name
item.module = this_mod
this_mod.items.by_name[item.name] = item
else
package = package .. '.'
display_name = summary
end
self.modules:append(this_mod)
this_mod.package = package
this_mod.mod_name = mname
this_mod.kinds = ModuleMap() -- the iterator over the module contents
else -- add the item to the module's item list
if this_mod then
-- new-style modules will have qualified names like 'mod.foo'
local mod,fname = split_dotted_name(item.name)
-- warning for inferred unqualified names in new style modules
-- (retired until we handle methods like Set:unset() properly)
if not mod and not this_mod.old_style and item.inferred then
--item:warning(item.name .. ' is declared in global scope')
end
-- the function may be qualified with a module alias...
if this_mod.tags.alias and mod == this_mod.tags.alias then
mod = this_mod.mod_name
end
-- if that's the mod_name, then we want to only use 'foo'
if mod == this_mod.mod_name and this_mod.tags.pragma ~= 'nostrip' then
item.name = fname
end
item.module = this_mod
local these_items = this_mod.items
these_items.by_name[item.name] = item
these_items:append(item)
-- register this item with the iterator
this_mod.kinds:add(item,these_items)
else
-- must be a free-standing function (sometimes a problem...)
item.display_name = display_name
this_mod.section = item
this_mod.kinds:add_kind(display_name,display_name)
end
else
-- add the item to the module's item list
if this_mod then
-- new-style modules will have qualified names like 'mod.foo'
local mod,fname = split_dotted_name(item.name)
-- warning for inferred unqualified names in new style modules
-- (retired until we handle methods like Set:unset() properly)
if not mod and not this_mod.old_style and item.inferred then
--item:warning(item.name .. ' is declared in global scope')
end
end
end
-- the function may be qualified with a module alias...
if this_mod.tags.alias and mod == this_mod.tags.alias then
mod = this_mod.mod_name
end
-- if that's the mod_name, then we want to only use 'foo'
if mod == this_mod.mod_name and this_mod.tags.pragma ~= 'nostrip' then
item.name = fname
end
-- right, this item was within a section or a 'class'
if this_mod.section then
item.section = this_mod.section.display_name
-- if it was a class, then the name should be 'Class.foo'
if this_mod.section.type == 'type' then
item.name = this_mod.section.name .. '.' .. item.name
end
else -- otherwise, just goes into the default sections (Functions,Tables,etc)
item.section = item.type
end
item.module = this_mod
local these_items = this_mod.items
these_items.by_name[item.name] = item
these_items:append(item)
-- register this item with the iterator
this_mod.kinds:add(item,these_items)
else
-- must be a free-standing function (sometimes a problem...)
end
end
end
end
function Item:_init(tags)
self.summary = tags.summary
self.description = tags.description
tags.summary = nil
tags.description = nil
self.tags = {}
self.formal_args = tags.formal_args
tags.formal_args = nil
for tag,value in pairs(tags) do
local ttype = known_tags[tag]
if ttype == TAG_MULTI then
if type(value) == 'string' then
value = List{value}
end
self.tags[tag] = value
elseif ttype == TAG_ID then
if type(value) ~= 'string' then
-- such tags are _not_ multiple, e.g. name
self:error(tag..' cannot have multiple values')
else
self.tags[tag] = tools.extract_identifier(value)
end
elseif ttype == TAG_SINGLE then
self.tags[tag] = value
else
self:warning ('unknown tag: '..tag)
end
end
self.summary = tags.summary
self.description = tags.description
tags.summary = nil
tags.description = nil
self.tags = {}
self.formal_args = tags.formal_args
tags.formal_args = nil
for tag,value in pairs(tags) do
local ttype = known_tags[tag]
if ttype == TAG_MULTI then
if type(value) == 'string' then
value = List{value}
end
self.tags[tag] = value
elseif ttype == TAG_ID then
if type(value) ~= 'string' then
-- such tags are _not_ multiple, e.g. name
self:error(tag..' cannot have multiple values')
else
self.tags[tag] = tools.extract_identifier(value)
end
elseif ttype == TAG_SINGLE then
self.tags[tag] = value
else
self:warning ('unknown tag: '..tag)
end
end
end
-- preliminary processing of tags. We check for any aliases, and for tags
-- which represent types. This implements the shortcut notation.
function Item.check_tag(tags,tag)
tag = doc.get_alias(tag) or tag
local ttype = known_tags[tag]
if ttype == TAG_TYPE then
tags.class = tag
tag = 'name'
end
return tag
tag = doc.get_alias(tag) or tag
local ttype = known_tags[tag]
if ttype == TAG_TYPE then
tags.class = tag
tag = 'name'
end
return tag
end
function Item:finish()
local tags = self.tags
self.name = tags.name
self.type = tags.class
self.usage = tags.usage
tags.name = nil
tags.class = nil
tags.usage = nil
-- see tags are multiple, but they may also be comma-separated
if tags.see then
tags.see = tools.expand_comma_list(tags.see)
end
if doc.project_level(self.type) then
-- we are a module, so become one!
self.items = List()
self.items.by_name = {}
setmetatable(self,Module)
else
-- params are either a function's arguments, or a table's fields, etc.
local params
if self.type == 'function' then
params = tags.param or List()
if tags['return'] then
self.ret = tags['return']
local tags = self.tags
self.name = tags.name
self.type = tags.class
self.usage = tags.usage
tags.name = nil
tags.class = nil
tags.usage = nil
-- see tags are multiple, but they may also be comma-separated
if tags.see then
tags.see = tools.expand_comma_list(tags.see)
end
if doc.project_level(self.type) then
-- we are a module, so become one!
self.items = List()
self.items.by_name = {}
setmetatable(self,Module)
elseif not doc.section_tag(self.type) then
-- params are either a function's arguments, or a table's fields, etc.
local params
if self.type == 'function' then
params = tags.param or List()
if tags['return'] then
self.ret = tags['return']
end
else
params = tags.field or List()
end
tags.param = nil
local names,comments = List(),List()
for p in params:iter() do
local name,comment = p:match('%s*([%w_%.:]+)(.*)')
names:append(name)
comments:append(comment)
end
-- not all arguments may be commented --
if self.formal_args then
-- however, ldoc allows comments in the arg list to be used
local fargs = self.formal_args
for a in fargs:iter() do
if not names:index(a) then
names:append(a)
comments:append (fargs.comments[a] or '')
end
else
params = tags.field or List()
end
tags.param = nil
local names,comments = List(),List()
for p in params:iter() do
local name,comment = p:match('%s*([%w_%.:]+)(.*)')
names:append(name)
comments:append(comment)
end
-- not all arguments may be commented --
if self.formal_args then
-- however, ldoc allows comments in the arg list to be used
local fargs = self.formal_args
for a in fargs:iter() do
if not names:index(a) then
names:append(a)
comments:append (fargs.comments[a] or '')
end
end
end
self.params = names
for i,name in ipairs(self.params) do
self.params[name] = comments[i]
end
self.args = '('..self.params:join(', ')..')'
end
end
end
self.params = names
for i,name in ipairs(self.params) do
self.params[name] = comments[i]
end
self.args = '('..self.params:join(', ')..')'
end
end
function Item:warning(msg)
local name = self.file and self.file.filename
if type(name) == 'table' then pretty.dump(name); name = '?' end
name = name or '?'
io.stderr:write(name,':',self.lineno or '?',' ',msg,'\n')
local name = self.file and self.file.filename
if type(name) == 'table' then pretty.dump(name); name = '?' end
name = name or '?'
io.stderr:write(name,':',self.lineno or '?',' ',msg,'\n')
end
-- resolving @see references. A word may be either a function in this module,
@ -244,96 +280,101 @@ end
-- If this isn't successful then try prepending the current package to the reference,
-- and try to to resolve this.
function Module:resolve_references(modules)
local found = List()
local found = List()
local function process_see_reference (item,see,s)
local mod_ref,fun_ref,name,packmod
-- is this a fully qualified module name?
local mod_ref = modules.by_name[s]
if mod_ref then return mod_ref,nil end
local packmod,name = split_dotted_name(s) -- e.g. 'pl.utils','split'
if packmod then -- qualified name
mod_ref = modules.by_name[packmod] -- fully qualified mod name?
if not mod_ref then
mod_ref = modules.by_name[self.package..packmod]
end
if not mod_ref then
item:warning("module not found: "..packmod)
return nil
end
fun_ref = mod_ref.items.by_name[name]
if fun_ref then return mod_ref,fun_ref
else
item:warning("function not found: "..s.." in "..mod_ref.name)
end
else -- plain jane name; module in this package, function in this module
mod_ref = modules.by_name[self.package..s]
if mod_ref then return mod_ref,nil end
fun_ref = self.items.by_name[s]
if fun_ref then return self,fun_ref
else
item:warning("function not found: "..s.." in this module")
end
end
end
local function process_see_reference (item,see,s)
local mod_ref,fun_ref,name,packmod
-- is this a fully qualified module name?
local mod_ref = modules.by_name[s]
if mod_ref then return mod_ref,nil end
local packmod,name = split_dotted_name(s) -- e.g. 'pl.utils','split'
if packmod then -- qualified name
mod_ref = modules.by_name[packmod] -- fully qualified mod name?
if not mod_ref then
mod_ref = modules.by_name[self.package..packmod]
end
if not mod_ref then
item:warning("module not found: "..packmod)
return nil
end
fun_ref = mod_ref.items.by_name[name]
if fun_ref then
return mod_ref,fun_ref
else
item:warning("function not found: "..s.." in "..mod_ref.name)
end
else -- plain jane name; module in this package, function in this module
mod_ref = modules.by_name[self.package..s]
if mod_ref then return mod_ref,nil end
fun_ref = self.items.by_name[s]
if fun_ref then return self,fun_ref
else
item:warning("function not found: "..s.." in this module")
end
end
end
for item in self.items:iter() do
local see = item.tags.see
if see then -- this guy has @see references
item.see = List()
for s in see:iter() do
local mod_ref, item_ref = process_see_reference(item,see,s)
if mod_ref then
local name = item_ref and item_ref.name or ''
item.see:append {mod=mod_ref.name,name=name,label=s}
found:append{item,s}
end
for item in self.items:iter() do
local see = item.tags.see
if see then -- this guy has @see references
item.see = List()
for s in see:iter() do
local mod_ref, item_ref = process_see_reference(item,see,s)
if mod_ref then
local name = item_ref and item_ref.name or ''
-- this is deeply hacky; classes have 'Class ' prepended.
if item_ref.type == 'type' then
name = 'Class_'..name
end
item.see:append {mod=mod_ref.name,name=name,label=s}
found:append{item,s}
end
end
end
-- mark as found, so we don't waste time re-searching
for f in found:iter() do
f[1].tags.see:remove_value(f[2])
end
end
end
end
-- mark as found, so we don't waste time re-searching
for f in found:iter() do
f[1].tags.see:remove_value(f[2])
end
end
-- make a text dump of the contents of this File object.
-- The level of detail is controlled by the 'verbose' parameter.
-- Primarily intended as a debugging tool.
function File:dump(verbose)
for mod in self.modules:iter() do
mod:dump(verbose)
end
for mod in self.modules:iter() do
mod:dump(verbose)
end
end
function Module:dump(verbose)
print '----'
print(self.type..':',self.name,self.summary)
if self.description then print(self.description) end
for item in self.items:iter() do
item:dump(verbose)
end
print '----'
print(self.type..':',self.name,self.summary)
if self.description then print(self.description) end
for item in self.items:iter() do
item:dump(verbose)
end
end
function Item:dump(verbose)
local tags = self.tags
local name = self.name
if self.type == 'function' then
name = name .. self.args
end
if verbose then
print(self.type,name)
print(self.summary)
if self.description then print(self.description) end
for _,p in ipairs(self.params) do
print(p,self.params[p])
end
for tag, value in pairs(self.tags) do
print(tag,value)
end
else
print('* '..name..' - '..self.summary)
end
local tags = self.tags
local name = self.name
if self.type == 'function' then
name = name .. self.args
end
if verbose then
print(self.type,name)
print(self.summary)
if self.description then print(self.description) end
for _,p in ipairs(self.params) do
print(p,self.params[p])
end
for tag, value in pairs(self.tags) do
print(tag,value)
end
else
print('* '..name..' - '..self.summary)
end
end
return doc

656
ldoc.lua
View File

@ -16,7 +16,7 @@ app.require_here()
local args = lapp [[
ldoc, a documentation generator for Lua, vs 0.2 Beta
-d,--dir (default docs) output directory
-o (default 'index') output name
-o,--output (default 'index') output name
-v,--verbose verbose
-q,--quiet suppress output
-m,--module module docs as text
@ -38,18 +38,21 @@ local KindMap = tools.KindMap
class.ModuleMap(KindMap)
function ModuleMap:_init ()
self.klass = ModuleMap
self.klass = ModuleMap
self.fieldname = 'section'
end
ModuleMap:add_kind('function','Functions','Parameters')
ModuleMap:add_kind('table','Tables','Fields')
ModuleMap:add_kind('field','Fields')
class.ProjectMap(KindMap)
ProjectMap.project_level = true
function ProjectMap:_init ()
self.klass = ProjectMap
self.klass = ProjectMap
self.fieldname = 'type'
end
ProjectMap:add_kind('module','Modules')
@ -59,36 +62,45 @@ ProjectMap:add_kind('script','Scripts')
-- the ldoc table represents the API available in `config.ld`.
local ldoc = {}
local add_language_extension
-- aliases to existing tags can be defined. E.g. just 'p' for 'param'
function ldoc.alias (a,tag)
doc.add_alias(a,tag)
doc.add_alias(a,tag)
end
function ldoc.add_language_extension(ext,lang)
add_language_extension(ext,lang)
end
function ldoc.add_section (name,title,subname)
ModuleMap:add_kind(name,title,subname)
end
-- new tags can be added, which can be on a project level.
function ldoc.new_type (tag,header,project_level)
doc.add_tag(tag,doc.TAG_TYPE,project_level)
if project_level then
ProjectMap:add_kind(tag,header)
else
ModuleMap:add_kind(tag,header)
end
doc.add_tag(tag,doc.TAG_TYPE,project_level)
if project_level then
ProjectMap:add_kind(tag,header)
else
ModuleMap:add_kind(tag,header)
end
end
-- any file called 'config.ld' found in the source tree will be
-- handled specially. It will be loaded using 'ldoc' as the environment.
local function read_ldoc_config (fname)
local directory = path.dirname(fname)
print('reading configuration from '..fname)
local txt = utils.readfile(fname)
-- Penlight defines loadin for Lua 5.1 as well
local chunk,err = loadin(ldoc,txt)
if chunk then
local ok
ok,err = pcall(chunk)
end
if err then print('error loading config file '..fname..': '..err) end
return directory
local directory = path.dirname(fname)
print('reading configuration from '..fname)
local txt = utils.readfile(fname)
-- Penlight defines loadin for Lua 5.1 as well
local chunk,err = loadin(ldoc,txt)
if chunk then
local ok
ok,err = pcall(chunk)
end
if err then print('error loading config file '..fname..': '..err) end
return directory
end
------ Parsing the Source --------------
@ -103,16 +115,16 @@ local luadoc_tag = '^%s*@(%a+)%s(.+)'
-- assumes that the doc comment consists of distinct tag lines
function parse_tags(text)
local lines = stringio.lines(text)
local preamble, line = tools.grab_while_not(lines,luadoc_tag)
local tag_items = {}
local follows
while line do
local tag,rest = line:match(luadoc_tag)
follows, line = tools.grab_while_not(lines,luadoc_tag)
append(tag_items,{tag, rest .. '\n' .. follows})
end
return preamble,tag_items
local lines = stringio.lines(text)
local preamble, line = tools.grab_while_not(lines,luadoc_tag)
local tag_items = {}
local follows
while line do
local tag,rest = line:match(luadoc_tag)
follows, line = tools.grab_while_not(lines,luadoc_tag)
append(tag_items,{tag, rest .. '\n' .. follows})
end
return preamble,tag_items
end
-- This takes the collected comment block, and uses the docstyle to
@ -121,25 +133,25 @@ end
-- If a tag appears more than once, then its value becomes a list of strings.
-- Alias substitution and @TYPE NAME shortcutting is handled by Item.check_tag
local function extract_tags (s)
if s:match '^%s*$' then return {} end
local preamble,tag_items = parse_tags(s)
local strip = tools.strip
local summary,description = preamble:match('^(.-[%.?])%s(.+)')
if not summary then summary = preamble end
local tags = {summary=summary and strip(summary),description=description and strip(description)}
for _,item in ipairs(tag_items) do
local tag,value = item[1],item[2]
tag = Item.check_tag(tags,tag)
value = strip(value)
local old_value = tags[tag]
if old_value then
if type(old_value)=='string' then tags[tag] = List{old_value} end
tags[tag]:append(value)
else
tags[tag] = value
end
end
return Map(tags)
if s:match '^%s*$' then return {} end
local preamble,tag_items = parse_tags(s)
local strip = tools.strip
local summary,description = preamble:match('^(.-[%.?])%s(.+)')
if not summary then summary = preamble end
local tags = {summary=summary and strip(summary),description=description and strip(description)}
for _,item in ipairs(tag_items) do
local tag,value = item[1],item[2]
tag = Item.check_tag(tags,tag)
value = strip(value)
local old_value = tags[tag]
if old_value then
if type(old_value)=='string' then tags[tag] = List{old_value} end
tags[tag]:append(value)
else
tags[tag] = value
end
end
return Map(tags)
end
local quit = utils.quit
@ -148,42 +160,42 @@ local quit = utils.quit
class.Lang()
function Lang:trim_comment (s)
return s:gsub(self.line_comment,'')
return s:gsub(self.line_comment,'')
end
function Lang:start_comment (v)
local line = v:match (self.start_comment_)
local block = v:match(self.block_comment)
return line or block, block
local line = v:match (self.start_comment_)
local block = v:match(self.block_comment)
return line or block, block
end
function Lang:empty_comment (v)
return v:match(self.empty_comment_)
return v:match(self.empty_comment_)
end
function Lang:grab_block_comment(v,tok)
v = v:gsub(self.block_comment,'')
return tools.grab_block_comment(v,tok,self.end_block1,self.end_block2)
v = v:gsub(self.block_comment,'')
return tools.grab_block_comment(v,tok,self.end_block1,self.end_block2)
end
function Lang:find_module(tok,t,v)
return '...',t,v
return '...',t,v
end
function Lang:function_follows(t,v)
return false
return false
end
function Lang:finalize()
self.empty_comment_ = self.start_comment_..'%s*$'
self.empty_comment_ = self.start_comment_..'%s*$'
end
function Lang:search_for_token (tok,type,value,t,v)
while t and not (t == type and v == value) do
if t == 'comment' and self:start_comment(v) then return nil,t,v end
t,v = tnext(tok)
end
return t ~= nil,t,v
while t and not (t == type and v == value) do
if t == 'comment' and self:start_comment(v) then return nil,t,v end
t,v = tnext(tok)
end
return t ~= nil,t,v
end
function Lang:parse_function_header (tags,tok,toks)
@ -196,18 +208,18 @@ end
class.Lua(Lang)
function Lua:_init()
self.line_comment = '^%-%-+' -- used for stripping
self.start_comment_ = '^%-%-%-+' -- used for doc comment line start
self.block_comment = '^%-%-%[%[%-+' -- used for block doc comments
self.end_block1 = ']'
self.end_block2 = ']'
self:finalize()
self.line_comment = '^%-%-+' -- used for stripping
self.start_comment_ = '^%-%-%-+' -- used for doc comment line start
self.block_comment = '^%-%-%[%[%-+' -- used for block doc comments
self.end_block1 = ']'
self.end_block2 = ']'
self:finalize()
end
function Lua.lexer(fname)
local f,e = io.open(fname)
if not f then quit(e) end
return lexer.lua(f,{}),f
local f,e = io.open(fname)
if not f then quit(e) end
return lexer.lua(f,{}),f
end
-- If a module name was not provided, then we look for an explicit module()
@ -215,36 +227,36 @@ end
-- we should go back and process it. Likewise, module(...) also means
-- that we must infer the module name.
function Lua:find_module(tok,t,v)
local res
res,t,v = self:search_for_token(tok,'iden','module',t,v)
if not res then return nil,t,v end
t,v = tnext(tok)
if t == '(' then t,v = tnext(tok) end
if t == 'string' then -- explicit name, cool
return v,t,v
elseif t == '...' then -- we have to guess!
return '...',t,v
end
local res
res,t,v = self:search_for_token(tok,'iden','module',t,v)
if not res then return nil,t,v end
t,v = tnext(tok)
if t == '(' then t,v = tnext(tok) end
if t == 'string' then -- explicit name, cool
return v,t,v
elseif t == '...' then -- we have to guess!
return '...',t,v
end
end
function Lua:function_follows(t,v)
return t == 'keyword' and v == 'function'
return t == 'keyword' and v == 'function'
end
function Lua:parse_function_header (tags,tok,toks)
tags.name = tools.get_fun_name(tok)
tags.formal_args = tools.get_parameters(toks)
tags.class = 'function'
tags.name = tools.get_fun_name(tok)
tags.formal_args = tools.get_parameters(toks)
tags.class = 'function'
end
function Lua:parse_extra (tags,tok,toks)
if tags.class == 'table' and not tags.fields then
local res,t,v = self:search_for_token(tok,'{','{',tok())
if not res then return nil,t,v end
tags.formal_args = tools.get_parameters(toks,'}',function(s)
return s == ',' or s == ';'
end)
end
if tags.class == 'table' and not tags.fields then
local res,t,v = self:search_for_token(tok,'{','{',tok())
if not res then return nil,t,v end
tags.formal_args = tools.get_parameters(toks,'}',function(s)
return s == ',' or s == ';'
end)
end
end
@ -253,21 +265,21 @@ local lua = Lua()
class.CC(Lang)
function CC:_init()
self.line_comment = '^//+'
self.start_comment_ = '^///+'
self.block_comment = '^/%*%*+'
self:finalize()
self.line_comment = '^//+'
self.start_comment_ = '^///+'
self.block_comment = '^/%*%*+'
self:finalize()
end
function CC.lexer(f)
f,err = utils.readfile(f)
if not f then quit(err) end
return lexer.cpp(f,{})
f,err = utils.readfile(f)
if not f then quit(err) end
return lexer.cpp(f,{})
end
function CC:grab_block_comment(v,tok)
v = v:gsub(self.block_comment,'')
return 'comment',v:sub(1,-3)
v = v:gsub(self.block_comment,'')
return 'comment',v:sub(1,-3)
end
local cc = CC()
@ -279,113 +291,118 @@ local cc = CC()
-- encountered, then ldoc looks for a call to module() to find the name of the
-- module.
local function parse_file(fname,lang)
local line,f = 1
local F = File(fname)
local module_found, first_comment = false,true
local line,f = 1
local F = File(fname)
local module_found, first_comment = false,true
local tok,f = lang.lexer(fname)
local toks = tools.space_skip_getter(tok)
local tok,f = lang.lexer(fname)
local toks = tools.space_skip_getter(tok)
function lineno () return lexer.lineno(tok) end
function filename () return fname end
function lineno () return lexer.lineno(tok) end
function filename () return fname end
function F:warning (msg,kind)
kind = kind or 'warning'
lineno() -- why is this necessary?
lineno()
io.stderr:write(kind..' '..fname..':'..lineno()..' '..msg,'\n')
end
function F:warning (msg,kind)
kind = kind or 'warning'
lineno() -- why is this necessary?
lineno()
io.stderr:write(kind..' '..fname..':'..lineno()..' '..msg,'\n')
end
function F:error (msg)
self:warning(msg,'error')
os.exit(1)
end
function F:error (msg)
self:warning(msg,'error')
os.exit(1)
end
local function add_module(tags,module_found,old_style)
tags.name = module_found
tags.class = 'module'
local item = F:new_item(tags,lineno())
item.old_style = old_style
end
local function add_module(tags,module_found,old_style)
tags.name = module_found
tags.class = 'module'
local item = F:new_item(tags,lineno())
item.old_style = old_style
end
local t,v = tok()
while t do
if t == 'comment' then
local comment = {}
local ldoc_comment,block = lang:start_comment(v)
if ldoc_comment and block then
t,v = lang:grab_block_comment(v,tok)
local t,v = tok()
while t do
if t == 'comment' then
local comment = {}
local ldoc_comment,block = lang:start_comment(v)
if ldoc_comment and block then
t,v = lang:grab_block_comment(v,tok)
end
if lang:empty_comment(v) then -- ignore rest of empty start comments
t,v = tok()
end
while t and t == 'comment' do
v = lang:trim_comment(v)
append(comment,v)
t,v = tok()
if t == 'space' and not v:match '\n' then
t,v = tok()
end
if lang:empty_comment(v) then -- ignore rest of empty start comments
t,v = tok()
end
while t and t == 'comment' do
v = lang:trim_comment(v)
append(comment,v)
t,v = tok()
end
if not t then break end -- no more file!
end
if not t then break end -- no more file!
if t == 'space' then t,v = tnext(tok) end
if t == 'space' then t,v = tnext(tok) end
local fun_follows,tags
if ldoc_comment or first_comment then
comment = table.concat(comment)
fun_follows = lang:function_follows(t,v)
if fun_follows or comment:find '@' or first_comment then
tags = extract_tags(comment)
-- handle the special case where the initial module comment was not
-- an ldoc style comment
if not ldoc_comment and first_comment and not tags.class then
tags.class = 'module'
ldoc_comment = true
F:warning 'Module doc comment assumed'
end
if doc.project_level(tags.class) then
module_found = tags.name
end
end
first_comment = false
local fun_follows,tags
if ldoc_comment or first_comment then
comment = table.concat(comment)
fun_follows = lang:function_follows(t,v)
if fun_follows or comment:find '@' or first_comment then
tags = extract_tags(comment)
-- handle the special case where the initial module comment was not
-- an ldoc style comment
if not ldoc_comment and first_comment and not tags.class then
tags.class = 'module'
ldoc_comment = true
F:warning 'Module doc comment assumed'
end
if doc.project_level(tags.class) then
module_found = tags.name
end
end
-- some hackery necessary to find the module() call
if not module_found and ldoc_comment then
local old_style
module_found,t,v = lang:find_module(tok,t,v)
-- right, we can add the module object ...
old_style = module_found ~= nil
if not module_found or module_found == '...' then
if not t then quit(fname..": end of file") end -- run out of file!
-- we have to guess the module name
module_found = tools.this_module_name(args.package,fname)
end
if not tags then tags = extract_tags(comment) end
add_module(tags,module_found,old_style)
tags = nil
-- if we did bump into a doc comment, then we can continue parsing it
first_comment = false
end
-- some hackery necessary to find the module() call
if not module_found and ldoc_comment then
local old_style
module_found,t,v = lang:find_module(tok,t,v)
-- right, we can add the module object ...
old_style = module_found ~= nil
if not module_found or module_found == '...' then
if not t then quit(fname..": end of file") end -- run out of file!
-- we have to guess the module name
module_found = tools.this_module_name(args.package,fname)
end
if not tags then tags = extract_tags(comment) end
add_module(tags,module_found,old_style)
tags = nil
-- if we did bump into a doc comment, then we can continue parsing it
end
-- end of a block of document comments
if ldoc_comment and tags then
if fun_follows then -- parse the function definition
lang:parse_function_header(tags,tok,toks)
else
lang:parse_extra(tags,tok,toks)
end
if tags.name then
F:new_item(tags,lineno()).inferred = fun_follows
end
-- end of a block of document comments
if ldoc_comment and tags then
if fun_follows then -- parse the function definition
lang:parse_function_header(tags,tok,toks)
else
lang:parse_extra(tags,tok,toks)
end
end
if t ~= 'comment' then t,v = tok() end
end
if f then f:close() end
return F
if tags.name then
F:new_item(tags,lineno()).inferred = fun_follows
end
end
end
if t ~= 'comment' then t,v = tok() end
end
if f then f:close() end
return F
end
function read_file(name,lang)
local F = parse_file(name,lang)
F:finish()
return F
local F = parse_file(name,lang)
F:finish()
return F
end
--- processing command line and preparing for output ---
@ -397,128 +414,134 @@ local multiple_files
local config_dir
local function extract_modules (F)
for mod in F.modules:iter() do
module_list:append(mod)
module_list.by_name[mod.name] = mod
end
for mod in F.modules:iter() do
module_list:append(mod)
module_list.by_name[mod.name] = mod
end
end
-- ldoc -m is expecting a Lua package; this converts this to a file path
if args.module then
local fullpath,lua = path.package_path(args.file)
if not fullpath then
local mpath,name = tools.split_dotted_name(args.file)
fullpath,lua = path.package_path(mpath)
if not fullpath then
quit("module "..args.file.." not found on module path")
else
args.module = name
end
end
if not lua then quit("module "..args.file.." is a binary extension") end
args.file = fullpath
local fullpath,lua = path.package_path(args.file)
if not fullpath then
local mpath,name = tools.split_dotted_name(args.file)
fullpath,lua = path.package_path(mpath)
if not fullpath then
quit("module "..args.file.." not found on module path")
else
args.module = name
end
end
if not lua then quit("module "..args.file.." is a binary extension") end
args.file = fullpath
end
if args.file == '.' then
args.file = lfs.currentdir()
args.file = lfs.currentdir()
else
args.file = path.abspath(args.file)
args.file = path.abspath(args.file)
end
local source_dir = args.file
if path.isfile(args.file) then
source_dir = path.splitpath(source_dir)
source_dir = path.splitpath(source_dir)
end
if args.package == '.' then
args.package = source_dir
args.package = source_dir
elseif args.package == '..' then
args.package = path.splitpath(source_dir)
args.package = path.splitpath(source_dir)
end
local file_types = {
['.lua'] = lua,
['.ldoc'] = lua,
['.luadoc'] = lua,
['.c'] = cc,
['.cpp'] = cc,
['.cxx'] = cc,
['.C'] = cc
['.lua'] = lua,
['.ldoc'] = lua,
['.luadoc'] = lua,
['.c'] = cc,
['.cpp'] = cc,
['.cxx'] = cc,
['.C'] = cc
}
function add_language_extension (ext,lang)
lang = (lang=='c' and cc) or (lang=='lua' and lua) or quit('unknown language')
if ext:sub(1,1) ~= '.' then ext = '.'..ext end
file_types[ext] = lang
end
local CONFIG_NAME = 'config.ld'
if path.isdir(args.file) then
local files = List(dir.getallfiles(args.file,'*.*'))
local config_files = files:filter(function(f)
return path.basename(f) == CONFIG_NAME
end)
local files = List(dir.getallfiles(args.file,'*.*'))
local config_files = files:filter(function(f)
return path.basename(f) == CONFIG_NAME
end)
-- finding more than one should probably be a warning...
if #config_files > 0 then
config_dir = read_ldoc_config(config_files[1])
end
-- finding more than one should probably be a warning...
if #config_files > 0 then
config_dir = read_ldoc_config(config_files[1])
end
for f in files:iter() do
local ext = path.extension(f)
local ftype = file_types[ext]
if ftype then
if args.verbose then print(path.basename(f)) end
local F = read_file(f,ftype)
file_list:append(F)
end
end
for F in file_list:iter() do
extract_modules(F)
end
multiple_files = true
for f in files:iter() do
local ext = path.extension(f)
local ftype = file_types[ext]
if ftype then
if args.verbose then print(path.basename(f)) end
local F = read_file(f,ftype)
file_list:append(F)
end
end
for F in file_list:iter() do
extract_modules(F)
end
multiple_files = true
elseif path.isfile(args.file) then
-- a single file may be accompanied by a config.ld in the same dir
local config_dir = path.dirname(args.file)
if config_dir == '' then config_dir = '.' end
local config = path.join(config_dir,CONFIG_NAME)
if path.isfile(config) then
read_ldoc_config(config)
end
local ext = path.extension(args.file)
local ftype = file_types[ext]
if not ftype then quit "unsupported extension" end
F = read_file(args.file,ftype)
extract_modules(F)
-- a single file may be accompanied by a config.ld in the same dir
local config_dir = path.dirname(args.file)
if config_dir == '' then config_dir = '.' end
local config = path.join(config_dir,CONFIG_NAME)
if path.isfile(config) then
read_ldoc_config(config)
end
local ext = path.extension(args.file)
local ftype = file_types[ext]
if not ftype then quit "unsupported extension" end
F = read_file(args.file,ftype)
extract_modules(F)
else
quit ("file or directory does not exist")
quit ("file or directory does not exist")
end
local project = ProjectMap()
for mod in module_list:iter() do
mod:resolve_references(module_list)
project:add(mod,module_list)
mod:resolve_references(module_list)
project:add(mod,module_list)
end
table.sort(module_list,function(m1,m2)
return m1.name < m2.name
return m1.name < m2.name
end)
-- ldoc -m will give a quick & dirty dump of the module's documentation;
-- using -v will make it more verbose
if args.module then
if #module_list == 0 then quit("no modules found") end
if args.module == true then
F:dump(args.verbose)
else
local fun = module_list[1].items.by_name[args.module]
if not fun then quit(args.module.." is not part of this module") end
fun:dump(true)
end
return
if #module_list == 0 then quit("no modules found") end
if args.module == true then
F:dump(args.verbose)
else
local fun = module_list[1].items.by_name[args.module]
if not fun then quit(args.module.." is not part of this module") end
fun:dump(true)
end
return
end
if args.dump then
for mod in module_list:iter() do
mod:dump(true)
end
os.exit()
for mod in module_list:iter() do
mod:dump(true)
end
os.exit()
end
local css, templ = 'ldoc.css','ldoc.ltp'
@ -530,7 +553,7 @@ if ldoc.style then args.style = path.join(config_dir,ldoc.style) end
-- '!' here means 'use same directory as the ldoc.lua script'
if args.style == '!' then
args.style = arg[0]:gsub('[^/\\]+$','')
args.style = arg[0]:gsub('[^/\\]+$','')
end
local module_template,err = utils.readfile (path.join(args.style,templ))
@ -540,66 +563,71 @@ if not module_template then quit(err) end
if ldoc.format then args.format = ldoc.format end
if args.format ~= 'plain' then
local ok,markup = pcall(require,args.format)
if not ok then quit("cannot load formatter: "..args.format) end
function ldoc.markup(txt)
if txt == nil then return '' end
txt = markup(txt)
return (txt:gsub('^%s*<p>(.+)</p>%s*$','%1'))
end
local ok,markup = pcall(require,args.format)
if not ok then quit("cannot load formatter: "..args.format) end
function ldoc.markup(txt)
if txt == nil then return '' end
txt = markup(txt)
return (txt:gsub('^%s*<p>(.+)</p>%s*$','%1'))
end
else
function ldoc.markup(txt)
return txt
end
function ldoc.markup(txt)
return txt
end
end
function generate_output()
-- ldoc.single = not multiple_files
local check_directory, check_file, writefile = tools.check_directory, tools.check_file, tools.writefile
ldoc.log = print
ldoc.kinds = project
ldoc.css = css
ldoc.modules = module_list
ldoc.title = ldoc.title or args.title
ldoc.project = ldoc.project or args.project
ldoc.single = not multiple_files
local check_directory, check_file, writefile = tools.check_directory, tools.check_file, tools.writefile
ldoc.log = print
ldoc.kinds = project
ldoc.css = css
ldoc.modules = module_list
ldoc.title = ldoc.title or args.title
ldoc.project = ldoc.project or args.project
local out,err = template.substitute(module_template,{ ldoc = ldoc })
if not out then quit(err) end
local out,err = template.substitute(module_template,{
ldoc = ldoc,
module = ldoc.single and ldoc.modules[1] or nil
})
if not out then quit(err) end
check_directory(args.dir)
check_directory(args.dir)
args.dir = args.dir .. path.sep
args.dir = args.dir .. path.sep
check_file(args.dir..css, path.join(args.style,css))
check_file(args.dir..css, path.join(args.style,css))
-- write out the module index
writefile(args.dir..'index.html',out)
-- write out the module index
writefile(args.dir..args.output..'.html',out)
-- write out the per-module documentation
ldoc.css = '../'..css
for kind, modules in project() do
kind = kind:lower()
check_directory(args.dir..kind)
for m in modules() do
-- write out the per-module documentation
if not ldoc.single then
ldoc.css = '../'..css
for kind, modules in project() do
kind = kind:lower()
check_directory(args.dir..kind)
for m in modules() do
out,err = template.substitute(module_template,{
module=m,
ldoc = ldoc
module=m,
ldoc = ldoc
})
if not out then
quit('template failed for '..m.name..': '..err)
quit('template failed for '..m.name..': '..err)
else
writefile(args.dir..kind..'/'..m.name..'.html',out)
writefile(args.dir..kind..'/'..m.name..'.html',out)
end
end
end
if not args.quiet then print('output written to '..args.dir) end
end
end
end
if not args.quiet then print('output written to '..args.dir) end
end
generate_output()
if args.verbose then
print 'modules'
for k in pairs(module_list.by_name) do print(k) end
print 'modules'
for k in pairs(module_list.by_name) do print(k) end
end

298
tools.lua
View File

@ -14,18 +14,18 @@ local quit = utils.quit
-- only over functions or tables, etc.
-- (something rather similar exists in LuaDoc)
function M.type_iterator (list,field,value)
return function()
local i = 1
return function()
local val = list[i]
while val and val[field] ~= value do
i = i + 1
val = list[i]
end
return function()
local i = 1
return function()
local val = list[i]
while val and val[field] ~= value do
i = i + 1
if val then return val end
end
end
val = list[i]
end
i = i + 1
if val then return val end
end
end
end
-- KindMap is used to iterate over a set of categories, called _kinds_,
@ -44,146 +44,154 @@ end
local KindMap = class()
M.KindMap = KindMap
-- calling a KindMap returns an iterator. This returns the kind, the iterator
-- over the items of that type, and the corresponding type.
-- calling a KindMap returns an iterator. This returns the kind and the iterator
-- over the items of that type.
function KindMap:__call ()
local i = 1
local klass = self.klass
return function()
local kind = klass.kinds[i]
if not kind then return nil end -- no more kinds
while not self[kind] do
i = i + 1
kind = klass.kinds[i]
if not kind then return nil end
end
i = i + 1
return kind, self[kind], klass.types_by_kind[kind]
end
local i = 1
local klass = self.klass
return function()
local kind = klass.kinds[i]
if not kind then return nil end -- no more kinds
while not self[kind] do
i = i + 1
kind = klass.kinds[i]
if not kind then return nil end
end
i = i + 1
local type = klass.types_by_kind [kind].type
return kind, self[kind], type
end
end
function KindMap:type_of (item)
local klass = self.klass
local kind = klass.types_by_tag[item.type]
return klass.types_by_kind [kind]
end
-- called for each new item. It does not actually create separate lists,
-- (although that would not break the interface) but creates iterators
-- for that item type if not already created.
function KindMap:add (item,items)
local kname = self.klass.types_by_tag[item.type]
if not self[kname] then
self[kname] = M.type_iterator (items,'type',item.type)
end
local group = item[self.fieldname]
local kname = self.klass.types_by_tag[group]
if not self[kname] then
self[kname] = M.type_iterator (items,self.fieldname,group)
end
end
-- KindMap has a 'class constructor' which is used to modify
-- any new base class.
function KindMap._class_init (klass)
klass.kinds = {} -- list in correct order of kinds
klass.types_by_tag = {} -- indexed by tag
klass.types_by_kind = {} -- indexed by kind
klass.kinds = {} -- list in correct order of kinds
klass.types_by_tag = {} -- indexed by tag
klass.types_by_kind = {} -- indexed by kind
end
function KindMap.add_kind (klass,tag,kind,subnames)
klass.types_by_tag[tag] = kind
klass.types_by_kind[kind] = {type=tag,subnames=subnames}
append(klass.kinds,kind)
klass.types_by_tag[tag] = kind
klass.types_by_kind[kind] = {type=tag,subnames=subnames}
append(klass.kinds,kind)
end
----- some useful utility functions ------
function M.module_basepath()
local lpath = List.split(package.path,';')
for p in lpath:iter() do
local p = path.dirname(p)
if path.isabs(p) then
return p
end
end
local lpath = List.split(package.path,';')
for p in lpath:iter() do
local p = path.dirname(p)
if path.isabs(p) then
return p
end
end
end
-- split a qualified name into the module part and the name part,
-- e.g 'pl.utils.split' becomes 'pl.utils' and 'split'
function M.split_dotted_name (s)
local s1,s2 = path.splitext(s)
if s2=='' then return nil
else return s1,s2:sub(2)
end
local s1,s2 = path.splitext(s)
if s2=='' then return nil
else return s1,s2:sub(2)
end
end
-- expand lists of possibly qualified identifiers
-- given something like {'one , two.2','three.drei.drie)'}
-- it will output {"one","two.2","three.drei.drie"}
function M.expand_comma_list (ls)
local new_ls = List()
for s in ls:iter() do
s = s:gsub('[^%.:%w]*$','')
if s:find ',' then
new_ls:extend(List.split(s,'%s*,%s*'))
else
new_ls:append(s)
end
end
return new_ls
local new_ls = List()
for s in ls:iter() do
s = s:gsub('[^%.:%w]*$','')
if s:find ',' then
new_ls:extend(List.split(s,'%s*,%s*'))
else
new_ls:append(s)
end
end
return new_ls
end
-- grab lines from a line iterator `iter` until the line matches the pattern.
-- Returns the joined lines and the line, which may be nil if we run out of
-- lines.
function M.grab_while_not(iter,pattern)
local line = iter()
local res = {}
while line and not line:match(pattern) do
append(res,line)
line = iter()
end
res = table.concat(res,'\n')
return res,line
local line = iter()
local res = {}
while line and not line:match(pattern) do
append(res,line)
line = iter()
end
res = table.concat(res,'\n')
return res,line
end
function M.extract_identifier (value)
return value:match('([%.:_%w]+)')
return value:match('([%.:_%w]+)')
end
function M.strip (s)
return s:gsub('^%s+',''):gsub('%s+$','')
return s:gsub('^%s+',''):gsub('%s+$','')
end
function M.check_directory(d)
if not path.isdir(d) then
lfs.mkdir(d)
end
if not path.isdir(d) then
lfs.mkdir(d)
end
end
function M.check_file (f,original)
if not path.exists(f) then
dir.copyfile(original,f)
end
if not path.exists(f) then
dir.copyfile(original,f)
end
end
function M.writefile(name,text)
local ok,err = utils.writefile(name,text)
if err then quit(err) end
local ok,err = utils.writefile(name,text)
if err then quit(err) end
end
function M.name_of (lpath)
lpath,ext = path.splitext(lpath)
return lpath
lpath,ext = path.splitext(lpath)
return lpath
end
function M.this_module_name (basename,fname)
local ext
if basename == '' then
--quit("module(...) needs package basename")
return M.name_of(fname)
end
basename = path.abspath(basename)
if basename:sub(-1,-1) ~= path.sep then
basename = basename..path.sep
end
local lpath,cnt = fname:gsub('^'..utils.escape(basename),'')
if cnt ~= 1 then quit("module(...) name deduction failed: base "..basename.." "..fname) end
lpath = lpath:gsub(path.sep,'.')
return M.name_of(lpath)
local ext
if basename == '' then
--quit("module(...) needs package basename")
return M.name_of(fname)
end
basename = path.abspath(basename)
if basename:sub(-1,-1) ~= path.sep then
basename = basename..path.sep
end
local lpath,cnt = fname:gsub('^'..utils.escape(basename),'')
if cnt ~= 1 then quit("module(...) name deduction failed: base "..basename.." "..fname) end
lpath = lpath:gsub(path.sep,'.')
return M.name_of(lpath)
end
@ -200,62 +208,62 @@ local function value_of (tok) return tok[2] end
-- param tags.
function M.get_parameters (tok,endtoken,delim)
local args = List()
args.comments = {}
local ltl = lexer.get_separated_list(tok,endtoken,delim)
local args = List()
args.comments = {}
local ltl = lexer.get_separated_list(tok,endtoken,delim)
if #ltl[1] == 0 then return args end -- no arguments
if #ltl[1] == 0 then return args end -- no arguments
local function set_comment (idx,tok)
local text = value_of(tok):gsub('%s*$','')
args.comments[args[idx]] = text
end
local function set_comment (idx,tok)
local text = value_of(tok):gsub('%s*$','')
args.comments[args[idx]] = text
end
for i = 1,#ltl do
local tl = ltl[i]
if type_of(tl[1]) == 'comment' then
if i > 1 then set_comment(i-1,tl[1]) end
if #tl > 1 then
args:append(value_of(tl[2]))
end
else
args:append(value_of(tl[1]))
end
if i == #ltl then
local last_tok = tl[#tl]
if #tl > 1 and type_of(last_tok) == 'comment' then
set_comment(i,last_tok)
end
end
end
for i = 1,#ltl do
local tl = ltl[i]
if type_of(tl[1]) == 'comment' then
if i > 1 then set_comment(i-1,tl[1]) end
if #tl > 1 then
args:append(value_of(tl[2]))
end
else
args:append(value_of(tl[1]))
end
if i == #ltl then
local last_tok = tl[#tl]
if #tl > 1 and type_of(last_tok) == 'comment' then
set_comment(i,last_tok)
end
end
end
return args
return args
end
-- parse a Lua identifier - contains names separated by . and :.
function M.get_fun_name (tok)
local res = {}
local _,name = tnext(tok)
_,sep = tnext(tok)
while sep == '.' or sep == ':' do
append(res,name)
append(res,sep)
_,name = tnext(tok)
_,sep = tnext(tok)
end
append(res,name)
return table.concat(res)
local res = {}
local _,name = tnext(tok)
_,sep = tnext(tok)
while sep == '.' or sep == ':' do
append(res,name)
append(res,sep)
_,name = tnext(tok)
_,sep = tnext(tok)
end
append(res,name)
return table.concat(res)
end
-- space-skipping version of token iterator
function M.space_skip_getter(tok)
return function ()
local t,v = tok()
while t and t == 'space' do
t,v = tok()
end
return t,v
end
return function ()
local t,v = tok()
while t and t == 'space' do
t,v = tok()
end
return t,v
end
end
-- an embarassing function. The PL Lua lexer does not do block comments
@ -263,17 +271,17 @@ end
-- do them properly in full-text mode, due to a ordering mistake.
-- So, we do what we can ;)
function M.grab_block_comment (v,tok,end1,end2)
local res = {v}
local t,last_v
repeat
last_v = v
t,v = tok()
append(res,v)
until last_v == end1 and v == end2
table.remove(res)
table.remove(res)
res = table.concat(res)
return 'comment',res
local res = {v}
local t,last_v
repeat
last_v = v
t,v = tok()
append(res,v)
until last_v == end1 and v == end2
table.remove(res)
table.remove(res)
res = table.concat(res)
return 'comment',res
end