commit 73e22a3c6ce0659b1f73cca92b0f4b0a37b4ce09
Author: steve donovan <steve.j.donovan@gmail.com>
Date:   Tue Apr 12 19:07:47 2011 +0200

    initial commit

diff --git a/doc.lua b/doc.lua
new file mode 100644
index 0000000..4ea12e6
--- /dev/null
+++ b/doc.lua
@@ -0,0 +1,329 @@
+------
+-- Defining the ldoc document model.
+
+
+require 'pl'
+
+local doc = {}
+
+local tools = require 'tools'
+local split_dotted_name = tools.split_dotted_name
+
+-- these are the basic tags known to ldoc. They come in several varieties:
+--  - tags with multiple values like 'param' (TAG_MULTI)
+--  - tags which are identifiers, like 'name' (TAG_ID)
+--  - tags with a single value, like 'release' (TAG_SINGLE)
+--  - tags which represent a type, like 'function' (TAG_TYPE)
+local known_tags = {
+    param = 'M', see = 'M', usage = 'M', ['return'] = 'M', field = 'M', author='M';
+    class = 'id', name = 'id', pragma = 'id';
+    copyright = 'S', description = 'S', release = 'S';
+    module = 'T', script = 'T',['function'] = 'T', table = 'T'
+}
+known_tags._alias = {}
+known_tags._project_level = {
+    module = true,
+    script = true
+}
+
+local TAG_MULTI,TAG_ID,TAG_SINGLE,TAG_TYPE = 'M','id','S','T'
+doc.TAG_MULTI,doc.TAG_ID,doc.TAG_SINGLE,doc.TAG_TYPE = TAG_MULTI,TAG_ID,TAG_SINGLE,TAG_TYPE
+
+-- add a new tag.
+function doc.add_tag(tag,type,project_level)
+    if not known_tags[tag] then
+        known_tags[tag] = type
+        known_tags._project_level[tag] = project_level
+    end
+end
+
+-- add an alias to an existing tag (exposed through ldoc API)
+function doc.add_alias (a,tag)
+    known_tags._alias[a] = tag
+end
+
+-- get the tag alias value, if it exists.
+function doc.get_alias(tag)
+    return known_tags._alias[tag]
+end
+
+-- is it a'project level' tag, such as 'module' or 'script'?
+function doc.project_level(tag)
+    return known_tags._project_level[tag]
+end
+
+-- we process each file, resulting in a File object, which has a list of Item objects.
+-- Items can be modules, scripts ('project level') or functions, tables, etc.
+-- (In the code 'module' refers to any project level tag.)
+-- When the File object is finalized, we specialize some items as modules which
+-- are 'container' types containing functions and tables, etc.
+
+local File = class()
+local Item = class()
+local Module = class(Item) -- a specialized kind of Item
+
+doc.File = File
+doc.Item = Item
+doc.Module = Module
+
+function File:_init(filename)
+    self.filename = filename
+    self.items = List()
+    self.modules = List()
+end
+
+function File:new_item(tags,line)
+    local item = Item(tags)
+    self.items:append(item)
+    item.file = self
+    item.lineno = line
+    return item
+end
+
+function File:finish()
+    local this_mod
+    local items = self.items
+    for item in items:iter() do
+        item:finish()
+        if doc.project_level(item.type) then
+            this_mod = item
+            -- if name is 'package.mod', then mod_name is 'mod'
+            local package,mname = split_dotted_name(this_mod.name)
+            if not package then
+                mname = this_mod.name
+                package = ''
+            else
+                package = package .. '.'
+            end
+            self.modules:append(this_mod)
+            this_mod.package = package
+            this_mod.mod_name = mname
+            this_mod.kinds = ModuleMap() -- the iterator over the module contents
+        else -- add the item to the module's item list
+            if this_mod then
+                -- new-style modules will have qualified names like 'mod.foo'
+                local mod,fname = split_dotted_name(item.name)
+                -- warning for inferred unqualified names in new style modules
+                -- (retired until we handle methods like Set:unset() properly)
+                if not mod and not this_mod.old_style and item.inferred then
+                    --item:warning(item.name .. ' is declared in global scope')
+                end
+                -- if that's the mod_name, then we want to only use 'foo'
+                if mod == this_mod.mod_name and this_mod.tags.pragma ~= 'nostrip' then
+                    item.name = fname
+                end
+                item.module = this_mod
+                local these_items = this_mod.items
+                these_items.by_name[item.name] = item
+                these_items:append(item)
+
+                -- register this item with the iterator
+                this_mod.kinds:add(item,these_items)
+
+            else
+                -- must be a free-standing function (sometimes a problem...)
+            end
+        end
+    end
+end
+
+function Item:_init(tags)
+    self.summary = tags.summary
+    self.description = tags.description
+    tags.summary = nil
+    tags.description = nil
+    self.tags = {}
+    self.formal_args = tags.formal_args
+    tags.formal_args = nil
+    for tag,value in pairs(tags) do
+        local ttype = known_tags[tag]
+        if ttype == TAG_MULTI then
+            if type(value) == 'string' then
+                value = List{value}
+            end
+            self.tags[tag] = value
+        elseif ttype == TAG_ID then
+            if type(value) ~= 'string' then
+                -- such tags are _not_ multiple, e.g. name
+                self:error(tag..' cannot have multiple values')
+            else
+                self.tags[tag] = tools.extract_identifier(value)
+            end
+        elseif ttype == TAG_SINGLE then
+            self.tags[tag] = value
+        else
+            self:warning ('unknown tag: '..tag)
+        end
+    end
+end
+
+-- preliminary processing of tags. We check for any aliases, and for tags
+-- which represent types. This implements the shortcut notation.
+function Item.check_tag(tags,tag)
+    tag = doc.get_alias(tag) or tag
+    local ttype = known_tags[tag]
+    if ttype == TAG_TYPE then
+        tags.class = tag
+        tag = 'name'
+    end
+    return tag
+end
+
+
+function Item:finish()
+    local tags = self.tags
+    self.name = tags.name
+    self.type = tags.class
+    self.usage = tags.usage
+    tags.name = nil
+    tags.class = nil
+    tags.usage = nil
+    -- see tags are multiple, but they may also be comma-separated
+    if tags.see then
+        tags.see = tools.expand_comma_list(tags.see)
+    end
+    if  doc.project_level(self.type) then
+        -- we are a module, so become one!
+        self.items = List()
+        self.items.by_name = {}
+        setmetatable(self,Module)
+    else
+        -- params are either a function's arguments, or a table's fields, etc.
+        local params
+        if self.type == 'function' then
+            params = tags.param or List()
+            if tags['return'] then
+                self.ret = tags['return']
+            end
+        else
+            params = tags.field or List()
+        end
+        tags.param = nil
+        local names,comments = List(),List()
+        for p in params:iter() do
+            local name,comment = p:match('%s*([%w_%.:]+)(.*)')
+            names:append(name)
+            comments:append(comment)
+        end
+        -- not all arguments may be commented --
+        if self.formal_args then
+            -- however, ldoc allows comments in the arg list to be used
+            local fargs = self.formal_args
+            for a in fargs:iter() do
+                if not names:index(a) then
+                    names:append(a)
+                    comments:append (fargs.comments[a] or '')
+                end
+            end
+        end
+        self.params = names
+        for i,name in ipairs(self.params) do
+            self.params[name] = comments[i]
+        end
+        self.args = '('..self.params:join(', ')..')'
+    end
+end
+
+function Item:warning(msg)
+    local name = self.file and self.file.filename
+    if type(name) == 'table' then pretty.dump(name); name = '?' end
+    name = name or '?'
+    io.stderr:write(name,':',self.lineno or '?',' ',msg,'\n')
+end
+
+-- resolving @see references. A word may be either a function in this module,
+-- or a module in this package. A MOD.NAME reference is within this package.
+-- Otherwise, the full qualified name must be used.
+-- First, check whether it is already a fully qualified module name.
+-- Then split it and see if the module part is a qualified module
+-- and try look up the name part in that module.
+-- If this isn't successful then try prepending the current package to the reference,
+-- and try to to resolve this.
+function Module:resolve_references(modules)
+    local found = List()
+
+    local function process_see_reference (item,see,s)
+        local mod_ref,fun_ref,name,packmod
+        -- is this a fully qualified module name?
+        local mod_ref = modules.by_name[s]
+        if mod_ref then return mod_ref,nil end
+        local packmod,name = split_dotted_name(s) -- e.g. 'pl.utils','split'
+        if packmod then -- qualified name
+            mod_ref = modules.by_name[packmod] -- fully qualified mod name?
+            if not mod_ref then
+                mod_ref = modules.by_name[self.package..packmod]
+            end
+            if not mod_ref then
+                item:warning("module not found: "..packmod)
+                return nil
+            end
+            fun_ref = mod_ref.items.by_name[name]
+            if fun_ref then return mod_ref,fun_ref
+            else
+                item:warning("function not found: "..s.." in "..mod_ref.name)
+            end
+        else -- plain jane name; module in this package, function in this module
+            mod_ref = modules.by_name[self.package..s]
+            if mod_ref then return mod_ref,nil end
+            fun_ref = self.items.by_name[s]
+            if fun_ref then return self,fun_ref
+            else
+                item:warning("function not found: "..s.." in this module")
+            end
+        end
+    end
+
+    for item in self.items:iter() do
+        local see = item.tags.see
+        if see then -- this guy has @see references
+            item.see = List()
+            for s in see:iter() do
+                local mod_ref, item_ref = process_see_reference(item,see,s)
+                if mod_ref then
+                    local name = item_ref and item_ref.name or ''
+                    item.see:append {mod=mod_ref.name,name=name,label=s}
+                    found:append{item,s}
+                end
+            end
+        end
+    end
+    -- mark as found, so we don't waste time re-searching
+    for f in found:iter() do
+        f[1].tags.see:remove_value(f[2])
+    end
+end
+
+-- make a text dump of the contents of this File object.
+-- The level of detail is controlled by the 'verbose' parameter.
+-- Primarily intended as a debugging tool.
+function File:dump(verbose)
+    for mod in self.modules:iter() do
+        print('Module:',mod.name,mod.summary,mod.description)
+        for item in mod.items:iter() do
+            item:dump(verbose)
+        end
+    end
+end
+
+function Item:dump(verbose)
+    local tags = self.tags
+    local name = self.name
+    if self.type == 'function' then
+        name = name .. self.args
+    end
+    if verbose then
+        print(self.type,name,self.summary)
+        print(self.description)
+        for p in self.params:iter() do
+            print(p,self.params[p])
+        end
+        for tag, value in pairs(self.tags) do
+            print(tag,value)
+        end
+    else
+        print('* '..name..' - '..self.summary)
+    end
+end
+
+return doc
+
diff --git a/ldoc.css b/ldoc.css
new file mode 100644
index 0000000..233fd2a
--- /dev/null
+++ b/ldoc.css
@@ -0,0 +1,289 @@
+/* BEGIN RESET
+
+Copyright (c) 2010, Yahoo! Inc. All rights reserved.
+Code licensed under the BSD License:
+http://developer.yahoo.com/yui/license.html
+version: 2.8.2r1
+*/
+html {
+    color: #000;
+    background: #FFF;
+}
+body,div,dl,dt,dd,ul,ol,li,h1,h2,h3,h4,h5,h6,pre,code,form,fieldset,legend,input,button,textarea,p,blockquote,th,td {
+    margin: 0;
+    padding: 0;
+}
+table {
+    border-collapse: collapse;
+    border-spacing: 0;
+}
+fieldset,img {
+    border: 0;
+}
+address,caption,cite,code,dfn,em,strong,th,var,optgroup {
+    font-style: inherit;
+    font-weight: inherit;
+}
+del,ins {
+    text-decoration: none;
+}
+li {
+    list-style: bullet;
+    margin-left: 20px;
+}
+caption,th {
+    text-align: left;
+}
+h1,h2,h3,h4,h5,h6 {
+    font-size: 100%;
+    font-weight: bold;
+}
+q:before,q:after {
+    content: '';
+}
+abbr,acronym {
+    border: 0;
+    font-variant: normal;
+}
+sup {
+    vertical-align: baseline;
+}
+sub {
+    vertical-align: baseline;
+}
+legend {
+    color: #000;
+}
+input,button,textarea,select,optgroup,option {
+    font-family: inherit;
+    font-size: inherit;
+    font-style: inherit;
+    font-weight: inherit;
+}
+input,button,textarea,select {*font-size:100%;
+}
+/* END RESET */
+
+body {
+    margin-left: 1em;
+    margin-right: 1em;
+    font-family: arial, helvetica, geneva, sans-serif;
+    background-color: #ffffff; margin: 0px;
+}
+
+code, tt { font-family: monospace; }
+
+body, p, td, th { font-size: .95em; line-height: 1.2em;}
+
+p, ul { margin: 10px 0 0 10px;}
+
+strong { font-weight: bold;}
+
+em { font-style: italic;}
+
+h1 {
+    font-size: 1.5em;
+    margin: 0 0 20px 0;
+}
+h2, h3, h4 { margin: 15px 0 10px 0; }
+h2 { font-size: 1.25em; }
+h3 { font-size: 1.15em; }
+h4 { font-size: 1.06em; }
+
+a:link { font-weight: bold; color: #004080; text-decoration: none; }
+a:visited { font-weight: bold; color: #006699; text-decoration: none; }
+a:link:hover { text-decoration: underline; }
+
+hr {
+    color:#cccccc;
+    background: #00007f;
+    height: 1px;
+}
+
+blockquote { margin-left: 3em; }
+
+ul { list-style-type: disc; }
+
+p.name {
+    font-family: "Andale Mono", monospace;
+    padding-top: 1em;
+}
+
+pre.example {
+    background-color: rgb(245, 245, 245);
+    border: 1px solid silver;
+    padding: 10px;
+    margin: 10px 0 10px 0;
+    font-family: "Andale Mono", monospace;
+    font-size: .85em;
+}
+
+table.index { border: 1px #00007f; }
+table.index td { text-align: left; vertical-align: top; }
+
+#container {
+    margin-left: 1em;
+    margin-right: 1em;
+    background-color: #f0f0f0;
+}
+
+#product {
+    text-align: center;
+    border-bottom: 1px solid #cccccc;
+    background-color: #ffffff;
+}
+
+#product big {
+    font-size: 2em;
+}
+
+#main {
+    background-color: #f0f0f0;
+    border-left: 2px solid #cccccc;
+}
+
+#navigation {
+    float: left;
+    width: 18em;
+    vertical-align: top;
+    background-color: #f0f0f0;
+    overflow: visible;
+}
+
+#navigation h2 {
+    background-color:#e7e7e7;
+    font-size:1.1em;
+    color:#000000;
+    text-align: left;
+    padding:0.2em;
+    border-top:1px solid #dddddd;
+    border-bottom:1px solid #dddddd;
+}
+
+#navigation ul
+{
+    font-size:1em;
+    list-style-type: none;
+    margin: 1px 1px 10px 1px;
+}
+
+#navigation li {
+    text-indent: -1em;
+    display: block;
+    margin: 3px 0px 0px 22px;
+}
+
+#navigation li li a {
+    margin: 0px 3px 0px -1em;
+}
+
+#content {
+    margin-left: 18em;
+    padding: 1em;
+    border-left: 2px solid #cccccc;
+    border-right: 2px solid #cccccc;
+    background-color: #ffffff;
+}
+
+#about {
+    clear: both;
+    padding: 5px;
+    border-top: 2px solid #cccccc;
+    background-color: #ffffff;
+}
+
+@media print {
+    body {
+        font: 12pt "Times New Roman", "TimeNR", Times, serif;
+    }
+    a { font-weight: bold; color: #004080; text-decoration: underline; }
+
+    #main {
+        background-color: #ffffff;
+        border-left: 0px;
+    }
+
+    #container {
+        margin-left: 2%;
+        margin-right: 2%;
+        background-color: #ffffff;
+    }
+
+    #content {
+        padding: 1em;
+        background-color: #ffffff;
+    }
+
+    #navigation {
+        display: none;
+    }
+    pre.example {
+        font-family: "Andale Mono", monospace;
+        font-size: 10pt;
+        page-break-inside: avoid;
+    }
+}
+
+table.module_list td {
+    border-width: 1px;
+    padding: 3px;
+    border-style: solid;
+    border-color: #cccccc;
+}
+table.module_list td.name { background-color: #f0f0f0; }
+table.module_list td.summary { width: 100%; }
+
+table.file_list {
+    border-width: 1px;
+    border-style: solid;
+    border-color: #cccccc;
+    border-collapse: collapse;
+}
+table.file_list td {
+    border-width: 1px;
+    padding: 3px;
+    border-style: solid;
+    border-color: #cccccc;
+}
+
+table.file_list td.name { background-color: #f0f0f0; }
+
+table.file_list td.summary { width: 100%; }
+
+table.function_list {
+    border-width: 1px;
+    border-style: solid;
+    border-color: #cccccc;
+    border-collapse: collapse;
+}
+table.function_list td {
+    border-width: 1px;
+    padding: 3px;
+    border-style: solid;
+    border-color: #cccccc;
+}
+
+table.function_list td.name { background-color: #f0f0f0; }
+
+table.function_list td.summary { width: 100%; }
+
+table.table_list {
+    border-width: 1px;
+    border-style: solid;
+    border-color: #cccccc;
+    border-collapse: collapse;
+}
+table.table_list td {
+    border-width: 1px;
+    padding: 3px;
+    border-style: solid;
+    border-color: #cccccc;
+}
+
+table.table_list td.name { background-color: #f0f0f0; }
+
+table.table_list td.summary { width: 100%; }
+
+dl.table dt, dl.function dt {border-top: 1px solid #ccc; padding-top: 1em;}
+dl.table dd, dl.function dd {padding-bottom: 1em; margin: 10px 0 0 20px;}
+dl.table h3, dl.function h3 {font-size: .95em;}
diff --git a/ldoc.ltp b/ldoc.ltp
new file mode 100644
index 0000000..60e2249
--- /dev/null
+++ b/ldoc.ltp
@@ -0,0 +1,170 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+   "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html>
+<head>
+    <title>$(ldoc.title)</title>
+    <link rel="stylesheet" href="$(ldoc.css)" type="text/css" />
+</head>
+<body>
+
+<div id="container">
+
+<div id="product">
+	<div id="product_logo"></div>
+	<div id="product_name"><big><b></b></big></div>
+	<div id="product_description"></div>
+</div> <!-- id="product" -->
+
+<div id="main">
+
+# local iter = ldoc.modules.iter
+# local M = ldoc.markup
+
+<!-- Menu -->
+
+<div id="navigation">
+<h1>$(ldoc.project)</h1>
+<ul>
+  <li><a href="../index.html">Index</a></li>
+</ul>
+
+# for kind, mods, type in ldoc.kinds() do
+<h2>$(kind)</h2>
+<ul>
+# local this_mod = module and module.name
+# local base = ""
+# kind = kind:lower()
+# if module then
+#   if module.type ~= type.type then base = "../"..kind.."/" end
+# else base = kind.."/"
+# end
+# for mod in mods() do
+#  if mod.name == this_mod then
+  <li><strong>$(mod.name)</strong></li>
+# else
+  <li><a href="$(base..mod.name).html">$(mod.name)</a></li>
+# end
+#end
+</ul>
+# end
+
+</div>
+
+<div id="content">
+
+# if module then
+
+<h1>Module <code>$(module.name)</code></h1>
+
+# local function use_li(ls)
+#   if #ls > 1 then return '<li>','</li>' else return '','' end
+# end
+# local function display_name(item)
+#   if item.type == 'function' then return item.name..'&nbsp;'..item.args
+#   else return item.name end
+#  end
+
+<p>$(M(module.summary))</p>
+<p>$(M(module.description))</p>
+
+## -- bang out the tables of item types for this module (e.g Functions, Tables, etc)
+# for kind,items in module.kinds() do
+<h2>$(kind)</h2>
+<table class="function_list">
+#  for item in items() do
+	<tr>
+	<td class="name" nowrap><a href="#$(item.name)">$(display_name(item))</a></td>
+	<td class="summary">$(M(item.summary))</td>
+	</tr>
+#  end -- for items
+</table>
+#end -- for kinds
+
+<br/>
+<br/>
+
+# --- currently works for both Functions and Tables. The params field either contains
+# --- function parameters or table fields.
+# for kind, items, type in module.kinds() do
+    <h2><a name="$(kind)"></a>$(kind)</h2>
+    <dl class="function">
+#  for item in items() do
+    <dt>
+    <a name = "$(item.name)"></a>
+    <strong>$(display_name(item))</strong>
+    </dt>
+    <dd>
+    $(M(item.summary))
+    $(M(item.description))
+
+#  if item.params and #item.params > 0 then
+    <h3>$(type.subnames):</h3>
+    <ul>
+#   for p in iter(item.params) do
+       <li><code><em>$(p)</em></code>: $(M(item.params[p]))</li>
+#   end -- for
+    </ul>
+#   end -- if params
+
+#   if item.usage then
+#     local li,il = use_li(item.usage)
+    <h3>Usage:</h3>
+    <ul>
+#     for usage in iter(item.usage) do
+        $(li)<pre class="example">$(usage)</pre>$(il)
+#     end -- for
+    </ul>
+#   end -- if usage
+
+#   if item.ret then
+#     local li,il = use_li(item.ret)
+    <h3>Returns:</h3>
+    <ol>
+#     for r in iter(item.ret) do
+        $(li)$(M(r))$(il)
+#     end -- for
+    </ol>
+#   end -- if returns
+
+#   if item.see then
+#     local li,il = use_li(item.see)
+    <h3>see also:</h3>
+    <ul>
+#     for see in iter(item.see) do
+         $(li)<a href="$(see.mod).html#$(see.name)">$(see.label)</a>$(il)
+#    end -- for
+    </ul>
+#   end -- if see
+</dd>
+# end -- for items
+</dl>
+# end -- for kinds
+
+# else -- if module
+
+# if ldoc.description then
+  <p>$(M(ldoc.description))</p>
+# end
+
+# for kind, mods, type in ldoc.kinds() do
+<h2>$(kind)</h2>
+# kind = kind:lower()
+# for m in mods() do
+<table class="module_list">
+	<tr>
+		<td class="name"><a href="$(kind)/$(m.name).html">$(m.name)</a></td>
+		<td class="summary">$(M(m.summary))</td>
+	</tr>
+#  end -- for modules
+</table>
+# end -- for kinds
+# end -- if module
+
+</div> <!-- id="content" -->
+</div> <!-- id="main" -->
+<div id="about">
+</div> <!-- id="about" -->
+</div> <!-- id="container" -->
+</body>
+</html>
+
diff --git a/ldoc.lua b/ldoc.lua
new file mode 100644
index 0000000..d50b09c
--- /dev/null
+++ b/ldoc.lua
@@ -0,0 +1,505 @@
+---------------
+-- ldoc, a Lua documentation generator.
+-- Compatible with luadoc-style annoations, but providing
+-- easier customization options. C/C++ support is provided.
+-- Steve Donovan, 2011
+
+require 'pl'
+
+local append = table.insert
+local template = require 'pl.template'
+local lapp = require 'pl.lapp'
+
+-- so we can find our private modules
+app.require_here()
+
+local args = lapp [[
+ldoc, a Lua documentation generator, vs 0.1 Beta
+  -d,--dir (default .) output directory
+  -o  (default 'index') output name
+  -v,--verbose          verbose
+  -q,--quiet            suppress output
+  -m,--module           module docs as text
+  -s,--style (default !) directory for templates and style
+  -p,--project (default ldoc) project name
+  -t,--title (default Reference) page title
+  -f,--format (default plain) formatting - can be markdown
+  -b,--package  (default '') top-level package basename (needed for module(...))
+  <file> (string) source file or directory containing source
+]]
+
+local lexer = require 'lexer'
+local doc = require 'doc'
+local Item,File,Module = doc.Item,doc.File,doc.Module
+local tools = require 'tools'
+local KindMap = tools.KindMap
+
+class.ModuleMap(KindMap)
+
+function ModuleMap:_init ()
+    self.klass = ModuleMap
+end
+
+ModuleMap:add_kind('function','Functions','Parameters')
+ModuleMap:add_kind('table','Tables','Fields')
+ModuleMap:add_kind('field','Fields')
+
+class.ProjectMap(KindMap)
+ProjectMap.project_level = true
+
+function ProjectMap:_init ()
+    self.klass = ProjectMap
+end
+
+ProjectMap:add_kind('module','Modules')
+ProjectMap:add_kind('script','Scripts')
+
+------- ldoc external API ------------
+
+-- the ldoc table represents the API available in `config.ld`.
+local ldoc = {}
+
+-- aliases to existing tags can be defined. E.g. just 'p' for 'param'
+function ldoc.alias (a,tag)
+    doc.add_alias(a,tag)
+end
+
+-- new tags can be added, which can be on a project level.
+function ldoc.new_type (tag,header,project_level)
+    doc.add_tag(tag,doc.TAG_TYPE,project_level)
+    if project_level then
+        ProjectMap:add_kind(tag,header)
+    else
+        ModuleMap:add_kind(tag,header)
+    end
+end
+
+-- any file called 'config.ld' found in the source tree will be
+-- handled specially. It will be loaded using 'ldoc' as the environment.
+local function read_ldoc_config (fname)
+    local directory = path.dirname(fname)
+    print('reading configuration from '..fname)
+    local txt = utils.readfile(fname)
+    -- Penlight defines loadin for Lua 5.1 as well
+    local chunk,err = loadin(ldoc,txt)
+    if chunk then
+        local ok
+        ok,err = pcall(chunk)
+    end
+    if err then print('error loading config file '..fname..': '..err) end
+    return directory
+end
+
+------ Parsing the Source --------------
+-- This uses the lexer from PL, but it should be possible to use Peter Odding's
+-- excellent Lpeg based lexer instead.
+
+local tnext = lexer.skipws
+
+-- This rather nasty looking code takes the collected comment block, and splits
+-- it up using '@', so it is specific to the LuaDoc style of commenting.
+-- If a tag appears more than once, then its value becomes a list of strings.
+-- Alias substitution and @TYPE NAME shortcutting is handled by Item.check_tag
+local function extract_tags (s)
+    if s:match '^%s*$' then return {} end
+    local strip = tools.strip
+    local items = utils.split(s,'@')
+    local summary,description = items[1]:match('^(.-)%.%s(.+)')
+    if not summary then summary = items[1] end
+    summary = summary .. '.'
+    table.remove(items,1)
+    local tags = {summary=summary and strip(summary),description=description and strip(description)}
+    for _,item in ipairs(items) do
+        local tag,value = item:match('(%a+)%s+(.+)%s*$')
+        if not tag then print(s); os.exit() end
+        tag = Item.check_tag(tags,tag)
+        value = strip(value)
+        local old_value = tags[tag]
+        if old_value then
+            if type(old_value)=='string' then tags[tag] = List{old_value} end
+            tags[tag]:append(value)
+        else
+            tags[tag] = value
+        end
+    end
+    return Map(tags)
+end
+
+local quit = utils.quit
+
+
+class.Lang()
+
+function Lang:trim_comment (s)
+    return s:gsub(self.line_comment,'')
+end
+
+function Lang:start_comment (v)
+    local line = v:match (self.start_comment_)
+    local block = v:match(self.block_comment)
+    return line or block, block
+end
+
+function Lang:empty_comment (v)
+    return v:match(self.empty_comment_)
+end
+
+function Lang:grab_block_comment(v,tok)
+    v = v:gsub(self.block_comment,'')
+    return tools.grab_block_comment(v,tok,self.end_block1,self.end_block2)
+end
+
+function Lang:find_module(tok)
+end
+
+function Lang:finalize()
+    self.empty_comment_ = self.start_comment_..'%s*$'
+end
+
+class.Lua(Lang)
+
+function Lua:_init()
+    self.line_comment = '^%-%-+' -- used for stripping
+    self.start_comment_ = '^%-%-%-+'     -- used for doc comment line start
+    self.block_comment = '^%-%-%[%[%-+' -- used for block doc comments
+    self.end_block1 = ']'
+    self.end_block2 = ']'
+    self:finalize()
+end
+
+function Lua.lexer(fname)
+    local f,e = io.open(fname)
+    if not f then quit(e) end
+    return lexer.lua(f,{}),f
+end
+
+-- If a module name was not provided, then we look for an explicit module()
+-- call. However, we should not try too hard; if we hit a doc comment then
+-- we should go back and process it. Likewise, module(...) also means
+-- that we must infer the module name.
+function Lua:find_module(tok,t,v)
+    while t and not (t == 'iden' and v == 'module') do
+        t,v = tnext(tok)
+        if t == 'comment' and self:start_comment(v) then return nil,t,v end
+    end
+    if not t then return nil end
+    t,v = tnext(tok)
+    if t == '(' then t,v = tnext(tok) end
+    if t == 'string' then -- explicit name, cool
+        return v,t,v
+    elseif t == '...' then -- we have to guess!
+        return '...',t,v
+    end
+end
+
+local lua = Lua()
+
+class.CC(Lang)
+
+function CC:_init()
+    self.line_comment = '^//+'
+    self.start_comment_ = '^///+'
+    self.block_comment = '^/%*%*+'
+    self:finalize()
+end
+
+function CC.lexer(f)
+    f,err = utils.readfile(f)
+    if not f then quit(err) end
+    return lexer.cpp(f,{})
+end
+
+function CC:grab_block_comment(v,tok)
+    v = v:gsub(self.block_comment,'')
+    return 'comment',v:sub(1,-3)
+end
+
+local cc = CC()
+
+
+-- parses a Lua file, looking for ldoc comments. These are like LuaDoc comments;
+-- they start with multiple '-'. If they don't define a name tag, then by default
+-- it is assumed that a function definition follows. If it is the first comment
+-- encountered, then ldoc looks for a call to module() to find the name of the
+-- module.
+local function parse_file(fname,lang)
+    local line,f = 1
+    local F = File(fname)
+    local module_found
+
+    local tok,f = lang.lexer(fname)
+    local toks = tools.space_skip_getter(tok)
+
+    function lineno () return lexer.lineno(tok) end
+    function filename () return fname end
+
+    function F:warning (msg,kind)
+        kind = kind or 'warning'
+        io.stderr:write(kind..' '..file..':'..lineno()..' '..msg,'\n')
+    end
+
+    function F:error (msg)
+        self:warning(msg,'error')
+        os.exit(1)
+    end
+
+    local t,v = tok()
+    while t do
+        if t == 'comment' then
+            local comment = {}
+            local ldoc_comment,block = lang:start_comment(v)
+            if ldoc_comment and block then
+                t,v = lang:grab_block_comment(v,tok)
+            end
+            if lang:empty_comment(v)  then -- ignore rest of empty start comments
+                t,v = tok()
+            end
+            while t and t == 'comment' do
+                v = lang:trim_comment(v)
+                append(comment,v)
+                t,v = tok()
+            end
+            if not t then break end -- no more file!
+
+            if t == 'space' then t,v = tnext(tok) end
+
+            local fun_follows,tags
+
+            if ldoc_comment then
+                comment = table.concat(comment)
+                if t == 'keyword' and v == 'local' then
+                    t,v = tnext(tok)
+                end
+                fun_follows = t == 'keyword' and v == 'function'
+                if fun_follows or comment:find '@' then
+                    tags = extract_tags(comment)
+                    if doc.project_level(tags.class) then
+                        module_found = tags.name
+                    end
+                end
+            end
+            -- some hackery necessary to find the module() call
+            if not module_found then
+                local old_style
+                module_found,t,v = lang:find_module(tok,t,v)
+                -- right, we can add the module object ...
+                old_style = module_found ~= nil
+                if not module_found or module_found == '...' then
+                    if not t then return end -- run out of file!
+                    -- we have to guess the module name
+                    module_found = tools.this_module_name(args.package,fname)
+                end
+                if not tags then tags = extract_tags(comment) end
+                tags.name = module_found
+                tags.class = 'module'
+                local item = F:new_item(tags,lineno())
+                item.old_style = old_style
+                tags = nil
+                -- if we did bump into a doc comment, then we can continue parsing it
+            end
+
+            -- end of a group of comments (may be just one)
+            if ldoc_comment and tags then
+                -- ldoc block
+                if fun_follows then -- parse the function definition
+                    tags.name = tools.get_fun_name(tok)
+                    tags.formal_args = tools.get_parameters(toks)
+                    tags.class = 'function'
+                end
+                if tags.name then --pretty.dump(tags)
+                    F:new_item(tags,lineno()).inferred = fun_follows
+                end
+            end
+        end
+        if t ~= 'comment' then t,v = tok() end
+    end
+    if f then f:close() end
+    return F
+end
+
+function read_file(name,lang)
+    local F = parse_file(name,lang)
+    F:finish()
+    return F
+end
+
+--- processing command line and preparing for output ---
+
+local F
+local file_list,module_list = List(),List()
+module_list.by_name = {}
+local multiple_files
+local config_dir
+
+local function extract_modules (F)
+    for mod in F.modules:iter() do
+        module_list:append(mod)
+        module_list.by_name[mod.name] = mod
+    end
+end
+
+-- ldoc -m is expecting a Lua package; this converts this to a file path
+if args.module then
+    local fullpath,lua = path.package_path(args.file)
+    if not fullpath then quit("module "..args.file.." not found on module path") end
+    if not lua then quit("module "..args.file.." is a binary extension") end
+    args.file = fullpath
+end
+
+if args.package == '' then
+    args.package = path.splitpath(args.file)
+end
+
+local file_types = {
+    ['.lua'] = lua,
+    ['.ldoc'] = lua,
+    ['.luadoc'] = lua,
+    ['.c'] = cc,
+    ['.cpp'] = cc,
+    ['.cxx'] = cc,
+    ['.C'] = cc
+}
+
+local CONFIG_NAME = 'config.ld'
+
+if path.isdir(args.file) then
+    local files = List(dir.getallfiles(args.file,'*.*'))
+    local config_files = files:filter(function(f)
+        return path.basename(f) == CONFIG_NAME
+    end)
+
+    -- finding more than one should probably be a warning...
+    if #config_files > 0 then
+       config_dir = read_ldoc_config(config_files[1])
+    end
+
+    for f in files:iter() do
+        local ext = path.extension(f)
+        local ftype = file_types[ext]
+        if ftype then
+            if args.verbose then print(path.basename(f)) end
+            local F = read_file(f,ftype)
+            file_list:append(F)
+        end
+    end
+    for F in file_list:iter() do
+        extract_modules(F)
+    end
+    multiple_files = true
+elseif path.isfile(args.file) then
+    -- a single file may be accompanied by a config.ld in the same dir
+    local config_dir = path.dirname(args.file)
+    if config_dir == '' then config_dir = '.' end
+    local config = path.join(config_dir,CONFIG_NAME)
+    if path.isfile(config) then
+        read_ldoc_config(config)
+    end
+    local ext = path.extension(args.file)
+    local ftype = file_types[ext]
+    if not ftype then quit "unsupported extension" end
+    F = read_file(args.file,ftype)
+    extract_modules(F)
+else
+    quit ("file or directory does not exist")
+end
+
+local project = ProjectMap()
+
+for mod in module_list:iter() do
+    mod:resolve_references(module_list)
+    project:add(mod,module_list)
+end
+
+table.sort(module_list,function(m1,m2)
+    return m1.name < m2.name
+end)
+
+-- ldoc -m will give a quick & dirty dump of the module's documentation;
+-- using -v will make it more verbose
+if args.module then
+    if #module_list == 0 then quit("no modules found") end
+    F:dump(args.verbose)
+    return
+end
+
+local css, templ = 'ldoc.css','ldoc.ltp'
+
+-- the style directory for template and stylesheet can be specified
+-- either by command-line 'style' argument or by 'style' field in
+-- config.ld. Then it is relative to the location of that file.
+if ldoc.style then args.style = path.join(config_dir,ldoc.style) end
+
+-- '!' here means 'use same directory as the ldoc.lua script'
+if args.style == '!' then
+    args.style = arg[0]:gsub('[^/\\]+$','')
+end
+
+local module_template,err = utils.readfile (path.join(args.style,templ))
+if not module_template then quit(err) end
+
+-- can specify formatter in config.ld
+if ldoc.format then args.format = ldoc.format end
+
+if args.format ~= 'plain' then
+    local ok,markup = pcall(require,args.format)
+    if not ok then quit("cannot load formatter: "..args.format) end
+    function ldoc.markup(txt)
+        txt = markup(txt)
+        return (txt:gsub('^%s*<p>(.+)</p>%s*$','%1'))
+    end
+else
+    function ldoc.markup(txt)
+        return txt
+    end
+end
+
+function generate_output()
+   -- ldoc.single = not multiple_files
+    local check_directory, check_file, writefile = tools.check_directory, tools.check_file, tools.writefile
+    ldoc.log = print
+    ldoc.kinds = project
+    ldoc.css = css
+    ldoc.modules = module_list
+    ldoc.title = ldoc.title or args.title
+    ldoc.project = ldoc.project or args.project
+
+    local out,err = template.substitute(module_template,{ ldoc = ldoc })
+    if not out then quit(err) end
+
+    check_directory(args.dir)
+
+    args.dir = args.dir .. path.sep
+
+    check_file(args.dir..css, path.join(args.style,css))
+
+    -- write out the module index
+    writefile(args.dir..'index.html',out)
+
+    -- write out the per-module documentation
+    ldoc.css = '../'..css
+    for kind, modules in project() do
+        kind = kind:lower()
+        check_directory(args.dir..kind)
+        for m in modules() do
+            out,err = template.substitute(module_template,{
+                module=m,
+                ldoc = ldoc
+            })
+            if not out then
+                quit('template failed for '..m.name..': '..err)
+            else
+                writefile(args.dir..kind..'/'..m.name..'.html',out)
+            end
+        end
+    end
+    if not args.quiet then print('output written to '..args.dir) end
+end
+
+generate_output()
+
+if args.verbose then
+    print 'modules'
+    for k in pairs(module_list.by_name) do print(k) end
+end
+
+
diff --git a/lexer.lua b/lexer.lua
new file mode 100644
index 0000000..56a30c0
--- /dev/null
+++ b/lexer.lua
@@ -0,0 +1,461 @@
+--- Lexical scanner for creating a sequence of tokens from text. <br>
+-- <p><code>lexer.scan(s)</code> returns an iterator over all tokens found in the
+-- string <code>s</code>. This iterator returns two values, a token type string
+-- (such as 'string' for quoted string, 'iden' for identifier) and the value of the
+-- token.
+-- <p>
+-- Versions specialized for Lua and C are available; these also handle block comments
+-- and classify keywords as 'keyword' tokens. For example:
+-- <pre class=example>
+-- > s = 'for i=1,n do'
+-- > for t,v in lexer.lua(s)  do print(t,v) end
+-- keyword for
+-- iden    i
+-- =       =
+-- number  1
+-- ,       ,
+-- iden    n
+-- keyword do
+-- </pre>
+-- See the Guide for further <a href="../../index.html#lexer">discussion</a> <br>
+-- @class module
+-- @name pl.lexer
+
+local yield,wrap = coroutine.yield,coroutine.wrap
+local strfind = string.find
+local strsub = string.sub
+local append = table.insert
+--[[
+module ('pl.lexer',utils._module)
+]]
+
+local function assert_arg(idx,val,tp)
+    if type(val) ~= tp then
+        error("argument "..idx.." must be "..tp, 2)
+    end
+end
+
+local lexer = {}
+
+local NUMBER1 = '^[%+%-]?%d+%.?%d*[eE][%+%-]?%d+'
+local NUMBER2 = '^[%+%-]?%d+%.?%d*'
+local NUMBER3 = '^0x[%da-fA-F]+'
+local NUMBER4 = '^%d+%.?%d*[eE][%+%-]?%d+'
+local NUMBER5 = '^%d+%.?%d*'
+local IDEN = '^[%a_][%w_]*'
+local WSPACE = '^%s+'
+local STRING1 = "^'.-[^\\]'"
+local STRING2 = '^".-[^\\]"'
+local STRING3 = '^[\'"][\'"]'
+local PREPRO = '^#.-[^\\]\n'
+
+local plain_matches,lua_matches,cpp_matches,lua_keyword,cpp_keyword
+
+local function tdump(tok)
+    return yield(tok,tok)
+end
+
+local function ndump(tok,options)
+    if options and options.number then
+        tok = tonumber(tok)
+    end
+    return yield("number",tok)
+end
+
+-- regular strings, single or double quotes; usually we want them
+-- without the quotes
+local function sdump(tok,options)
+    if options and options.string then
+        tok = tok:sub(2,-2)
+    end
+    return yield("string",tok)
+end
+
+-- long Lua strings need extra work to get rid of the quotes
+local function sdump_l(tok,options)
+    if options and options.string then
+        tok = tok:sub(3,-3)
+    end
+    return yield("string",tok)
+end
+
+local function chdump(tok,options)
+    if options and options.string then
+        tok = tok:sub(2,-2)
+    end
+    return yield("char",tok)
+end
+
+local function cdump(tok)
+    return yield('comment',tok)
+end
+
+local function wsdump (tok)
+    return yield("space",tok)
+end
+
+local function pdump (tok)
+    return yield('prepro',tok)
+end
+
+local function plain_vdump(tok)
+    return yield("iden",tok)
+end
+
+local function lua_vdump(tok)
+    if lua_keyword[tok] then
+        return yield("keyword",tok)
+    else
+        return yield("iden",tok)
+    end
+end
+
+local function cpp_vdump(tok)
+    if cpp_keyword[tok] then
+        return yield("keyword",tok)
+    else
+        return yield("iden",tok)
+    end
+end
+
+--- create a plain token iterator from a string or file-like object.
+-- @param s the string
+-- @param matches an optional match table (set of pattern-action pairs)
+-- @param filter a table of token types to exclude, by default {space=true}
+-- @param options a table of options; by default, {number=true,string=true},
+-- which means convert numbers and strip string quotes.
+function lexer.scan (s,matches,filter,options)
+    --assert_arg(1,s,'string')
+    local file = type(s) ~= 'string' and s
+    filter = filter or {space=true}
+    options = options or {number=true,string=true}
+    if filter then
+        if filter.space then filter[wsdump] = true end
+        if filter.comments then
+            filter[cdump] = true
+        end
+    end
+    if not matches then
+        if not plain_matches then
+            plain_matches = {
+                {WSPACE,wsdump},
+                {NUMBER3,ndump},
+                {IDEN,plain_vdump},
+                {NUMBER1,ndump},
+                {NUMBER2,ndump},
+                {STRING3,sdump},
+                {STRING1,sdump},
+                {STRING2,sdump},
+                {'^.',tdump}
+            }
+        end
+        matches = plain_matches
+    end
+    function lex ()
+        local i1,i2,idx,res1,res2,tok,pat,fun,capt
+        local line = 1
+        if file then s = file:read()..'\n' end
+        local sz = #s
+        local idx = 1
+        --print('sz',sz)
+        while true do
+            for _,m in ipairs(matches) do
+                pat = m[1]
+                fun = m[2]
+                if fun == nil then print(pat); os.exit() end
+                i1,i2 = strfind(s,pat,idx)
+                if i1 then
+                    tok = strsub(s,i1,i2)
+                    idx = i2 + 1
+                    if not (filter and filter[fun]) then
+                        lexer.finished = idx > sz
+                        res1,res2 = fun(tok,options)
+                    end
+                    if res1 then
+                        local tp = type(res1)
+                        -- insert a token list
+                        if tp=='table' then
+                            yield('','')
+                            for _,t in ipairs(res1) do
+                                yield(t[1],t[2])
+                            end
+                        elseif tp == 'string' then -- or search up to some special pattern
+                            i1,i2 = strfind(s,res1,idx)
+                            if i1 then
+                                tok = strsub(s,i1,i2)
+                                idx = i2 + 1
+                                yield('',tok)
+                            else
+                                yield('','')
+                                idx = sz + 1
+                            end
+                            --if idx > sz then return end
+                        else
+                            yield(line,idx)
+                        end
+                    end
+                    if idx > sz then
+                        if file then
+                            --repeat -- next non-empty line
+                                line = line + 1
+                                s = file:read()
+                                if not s then return end
+                            --until not s:match '^%s*$'
+                            s = s .. '\n'
+                            idx ,sz = 1,#s
+                            break
+                        else
+                            return
+                        end
+                    else break end
+                end
+            end
+        end
+    end
+    return wrap(lex)
+end
+
+local function isstring (s)
+    return type(s) == 'string'
+end
+
+--- insert tokens into a stream.
+-- @param tok a token stream
+-- @param a1 a string is the type, a table is a token list and
+-- a function is assumed to be a token-like iterator (returns type & value)
+-- @param a2 a string is the value
+function lexer.insert (tok,a1,a2)
+    if not a1 then return end
+    local ts
+    if isstring(a1) and isstring(a2) then
+        ts = {{a1,a2}}
+    elseif type(a1) == 'function' then
+        ts = {}
+        for t,v in a1() do
+            append(ts,{t,v})
+        end
+    else
+        ts = a1
+    end
+    tok(ts)
+end
+
+--- get everything in a stream upto a newline.
+-- @param tok a token stream
+-- @return a string
+function lexer.getline (tok)
+    local t,v = tok('.-\n')
+    return v
+end
+
+--- get current line number. <br>
+-- Only available if the input source is a file-like object.
+-- @param tok a token stream
+-- @return the line number and current column
+function lexer.lineno (tok)
+    return tok(0)
+end
+
+--- get the rest of the stream.
+-- @param tok a token stream
+-- @return a string
+function lexer.getrest (tok)
+    local t,v = tok('.+')
+    return v
+end
+
+--- get the Lua keywords as a set-like table.
+-- So <code>res["and"]</code> etc would be <code>true</code>.
+-- @return a table
+function lexer.get_keywords ()
+    if not lua_keyword then
+        lua_keyword = {
+            ["and"] = true, ["break"] = true,  ["do"] = true,
+            ["else"] = true, ["elseif"] = true, ["end"] = true,
+            ["false"] = true, ["for"] = true, ["function"] = true,
+            ["if"] = true, ["in"] = true,  ["local"] = true, ["nil"] = true,
+            ["not"] = true, ["or"] = true, ["repeat"] = true,
+            ["return"] = true, ["then"] = true, ["true"] = true,
+            ["until"] = true,  ["while"] = true
+        }
+    end
+    return lua_keyword
+end
+
+
+--- create a Lua token iterator from a string or file-like object.
+-- Will return the token type and value.
+-- @param s the string
+-- @param filter a table of token types to exclude, by default {space=true,comments=true}
+-- @param options a table of options; by default, {number=true,string=true},
+-- which means convert numbers and strip string quotes.
+function lexer.lua(s,filter,options)
+    filter = filter or {space=true,comments=true}
+    lexer.get_keywords()
+    if not lua_matches then
+        lua_matches = {
+            {WSPACE,wsdump},
+            {NUMBER3,ndump},
+            {IDEN,lua_vdump},
+            {NUMBER4,ndump},
+            {NUMBER5,ndump},
+            {STRING3,sdump},
+            {STRING1,sdump},
+            {STRING2,sdump},
+            {'^%-%-.-\n',cdump},
+            {'^%[%[.+%]%]',sdump_l},
+            {'^%-%-%[%[.+%]%]',cdump},
+            {'^==',tdump},
+            {'^~=',tdump},
+            {'^<=',tdump},
+            {'^>=',tdump},
+            {'^%.%.%.',tdump},
+            {'^.',tdump}
+        }
+    end
+    return lexer.scan(s,lua_matches,filter,options)
+end
+
+--- create a C/C++ token iterator from a string or file-like object.
+-- Will return the token type type and value.
+-- @param s the string
+-- @param filter a table of token types to exclude, by default {space=true,comments=true}
+-- @param options a table of options; by default, {number=true,string=true},
+-- which means convert numbers and strip string quotes.
+function lexer.cpp(s,filter,options)
+    filter = filter or {comments=true}
+    if not cpp_keyword then
+        cpp_keyword = {
+            ["class"] = true, ["break"] = true,  ["do"] = true, ["sizeof"] = true,
+            ["else"] = true, ["continue"] = true, ["struct"] = true,
+            ["false"] = true, ["for"] = true, ["public"] = true, ["void"] = true,
+            ["private"] = true, ["protected"] = true, ["goto"] = true,
+            ["if"] = true, ["static"] = true,  ["const"] = true, ["typedef"] = true,
+            ["enum"] = true, ["char"] = true, ["int"] = true, ["bool"] = true,
+            ["long"] = true, ["float"] = true, ["true"] = true, ["delete"] = true,
+            ["double"] = true,  ["while"] = true, ["new"] = true,
+            ["namespace"] = true, ["try"] = true, ["catch"] = true,
+            ["switch"] = true, ["case"] = true, ["extern"] = true,
+            ["return"] = true,["default"] = true,['unsigned']  = true,['signed'] = true,
+            ["union"] =  true, ["volatile"] = true, ["register"] = true,["short"] = true,
+        }
+    end
+    if not cpp_matches then
+        cpp_matches = {
+            {WSPACE,wsdump},
+            {PREPRO,pdump},
+            {NUMBER3,ndump},
+            {IDEN,cpp_vdump},
+            {NUMBER4,ndump},
+            {NUMBER5,ndump},
+            {STRING3,sdump},
+            {STRING1,chdump},
+            {STRING2,sdump},
+            {'^//.-\n',cdump},
+            {'^/%*.-%*/',cdump},
+            {'^==',tdump},
+            {'^!=',tdump},
+            {'^<=',tdump},
+            {'^>=',tdump},
+            {'^->',tdump},
+            {'^&&',tdump},
+            {'^||',tdump},
+            {'^%+%+',tdump},
+            {'^%-%-',tdump},
+            {'^%+=',tdump},
+            {'^%-=',tdump},
+            {'^%*=',tdump},
+            {'^/=',tdump},
+            {'^|=',tdump},
+            {'^%^=',tdump},
+            {'^::',tdump},
+            {'^.',tdump}
+        }
+    end
+    return lexer.scan(s,cpp_matches,filter,options)
+end
+
+--- get a list of parameters separated by a delimiter from a stream.
+-- @param tok the token stream
+-- @param endtoken end of list (default ')'). Can be '\n'
+-- @param delim separator (default ',')
+-- @return a list of token lists.
+function lexer.get_separated_list(tok,endtoken,delim)
+    endtoken = endtoken or ')'
+    delim = delim or ','
+    local parm_values = {}
+    local level = 1 -- used to count ( and )
+    local tl = {}
+    local function tappend (tl,t,val)
+        val = val or t
+        append(tl,{t,val})
+    end
+    local is_end
+    if endtoken == '\n' then
+        is_end = function(t,val)
+            return t == 'space' and val:find '\n'
+        end
+    else
+        is_end = function (t)
+            return t == endtoken
+        end
+    end
+    local token,value
+    while true do
+        token,value=tok()
+        if not token then return nil,'EOS' end -- end of stream is an error!
+        if is_end(token,value) and level == 1 then
+            append(parm_values,tl)
+            break
+        elseif token == '(' then
+            level = level + 1
+            tappend(tl,'(')
+        elseif token == ')' then
+            level = level - 1
+            if level == 0 then -- finished with parm list
+                append(parm_values,tl)
+                break
+            else
+                tappend(tl,')')
+            end
+        elseif token == delim and level == 1 then
+            append(parm_values,tl) -- a new parm
+            tl = {}
+        else
+            tappend(tl,token,value)
+        end
+    end
+    return parm_values,{token,value}
+end
+
+--- get the next non-space token from the stream.
+-- @param tok the token stream.
+function lexer.skipws (tok)
+    local t,v = tok()
+    while t == 'space' do
+        t,v = tok()
+    end
+    return t,v
+end
+
+local skipws = lexer.skipws
+
+--- get the next token, which must be of the expected type.
+-- Throws an error if this type does not match!
+-- @param tok the token stream
+-- @param expected_type the token type
+-- @param no_skip_ws whether we should skip whitespace
+function lexer.expecting (tok,expected_type,no_skip_ws)
+    assert_arg(1,tok,'function')
+    assert_arg(2,expected_type,'string')
+    local t,v
+    if no_skip_ws then
+        t,v = tok()
+    else
+        t,v = skipws(tok)
+    end
+    if t ~= expected_type then error ("expecting "..expected_type,2) end
+    return v
+end
+
+return lexer
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..887f20a
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,68 @@
+# LDoc Lua Documentation Tool
+
+LDoc is intended to be compatible with [LuaDoc](http://luadoc.luaforge.net/manual.htm) and thus follows the pattern set by the various *Doc tools:
+
+    --- first function. Some description
+    -- @param p1 first parameter
+    -- @param p2 second parameter
+    function mod1.first_fun(p1,p2)
+    end
+
+Various tags such as `see` and `usage` are supported, and generally the names of functions and modules can be inferred from the code.  The project grew out of the documentation needs of Penlight (and not always getting satisfaction with LuaDoc) and depends on Penlight itself. This allowed me to _not_ write a lot of code.
+
+LDoc tries to be faithful to LuaDoc, but provides some extensions
+
+    --- zero function. Two new ldoc features here; item types
+    -- can be used directly as tags, and aliases for tags
+    -- can be defined in config.lp.
+    -- @function zero_fun
+    -- @p k1 first
+    -- @p k2 second
+
+If a file `config.lp` is found in the source, then it will be loaded as Lua data. For example:
+
+    title = "testmod docs"
+    project = "testmod"
+    alias("p","param")
+
+Extra tag types can be defined:
+
+    new_type("macro","Macros")
+
+And then used as any other tag:
+
+    -----
+    -- A useful macro. This is an example of a custom 'kind'.
+    -- @macro first_macro
+    -- @see second_function
+
+LDoc can process C/C++ files:
+
+    /***
+    Create a table with given array and hash slots.
+    @function createtable
+    @param narr initial array slots, default 0
+    @param nrec initial hash slots, default 0
+    */
+    static int l_createtable (lua_State *L) {
+    ....
+
+Both `/**` and `///` are recognized as starting a comment block.
+
+
+The command-line options are:
+
+
+    ldoc, a Lua documentation generator, vs 0.1 Beta
+      -d,--dir (default .) output directory
+      -o  (default 'index') output name
+      -v,--verbose          verbose
+      -q,--quiet            suppress output
+      -m,--module           module docs as text
+      -s,--style (default !) directory for templates and style
+      -p,--project (default ldoc) project name
+      -t,--title (default Reference) page title
+      -f,--format (default plain) formatting - can be markdown
+      -b,--package  (default '') top-level package basename (needed for module(...))
+      <file> (string) source file or directory containing source
+
diff --git a/tests/example/config.ld b/tests/example/config.ld
new file mode 100644
index 0000000..2bf75bb
--- /dev/null
+++ b/tests/example/config.ld
@@ -0,0 +1,12 @@
+-- ldoc configuration file
+title = "testmod docs"
+project = "testmod"
+
+description = [[
+This description applies to the project as a whole.
+]]
+
+alias("p","param")
+
+new_type("macro","Macros")
+
diff --git a/tests/example/mod1.lua b/tests/example/mod1.lua
new file mode 100644
index 0000000..dddc125
--- /dev/null
+++ b/tests/example/mod1.lua
@@ -0,0 +1,60 @@
+---------------------------
+-- Test module providing bonzo.dog.
+-- Rest is a longer description
+-- @class module
+-- @name mod1
+
+--- zero function. Two new ldoc features here; item types
+-- can be used directly as tags, and aliases for tags
+-- can be defined in config.lp.
+-- @function zero_fun
+-- @p k1 first
+-- @p k2 second
+
+--- first function. Some description
+-- @param p1 first parameter
+-- @param p2 second parameter
+function mod1.first_fun(p1,p2)
+end
+
+-------------------------
+-- second function.
+-- @param ... var args!
+function mod1.second_function(...)
+end
+
+------------
+-- third function. Can also provide parameter comments inline,
+-- provided they follow this pattern.
+function mod1.third_function(
+    alpha, -- correction A
+    beta, -- correction B
+    gamma -- factor C
+    )
+end
+
+-----
+-- A useful macro. This is an example of a custom 'kind'.
+-- @macro first_macro
+-- @see second_function
+
+---- general configuration table
+-- @table config
+-- @field A alpha
+-- @field B beta
+-- @field C gamma
+mod1.config = {
+    A = 1,
+    B = 2,
+    C = 3
+}
+
+--[[--
+Another function. Using a Lua block comment
+@param p a parameter
+]]
+function mod1.zero_function(p)
+end
+
+
+
diff --git a/tests/example/modtest.lua b/tests/example/modtest.lua
new file mode 100644
index 0000000..e59adc0
--- /dev/null
+++ b/tests/example/modtest.lua
@@ -0,0 +1,9 @@
+-------
+-- A script.
+-- Scripts are not containers in the sense that modules are,
+-- (although perhaps the idea of 'commands' could be adopted for some utilities)
+-- It allows any upfront script comments to be included in the
+-- documentation.
+-- @script modtest
+
+print 'hello, world'
diff --git a/tests/example/mylib.c b/tests/example/mylib.c
new file mode 100644
index 0000000..a5a3071
--- /dev/null
+++ b/tests/example/mylib.c
@@ -0,0 +1,62 @@
+/// A sample C extension.
+// Demonstrates using ldoc's C/C++ support. Can either use /// or /*** */ etc.
+// @module mylib
+#include <string.h>
+#include <math.h>
+
+// includes for Lua
+#include <lua.h>
+#include <lauxlib.h>
+#include <lualib.h>
+
+/***
+Create a table with given array and hash slots.
+@function createtable
+@param narr initial array slots, default 0
+@param nrec initial hash slots, default 0
+*/
+static int l_createtable (lua_State *L) {
+  int narr = luaL_optint(L,1,0);
+  int nrec = luaL_optint(L,2,0);
+  lua_createtable(L,narr,nrec);
+  return 1;
+}
+
+/***
+Solve a quadratic equation.
+@function solve
+@param a coefficient of x^2
+@param b coefficient of x
+@param c constant
+@return first root
+@return second root
+*/
+static int l_solve (lua_State *L) {
+    double a = lua_tonumber(L,1);  // coeff of x*x
+    double b = lua_tonumber(L,2);  // coef of x
+    double c = lua_tonumber(L,3);  // constant
+    double abc = b*b - 4*a*c;
+    if (abc < 0.0) {
+        lua_pushnil(L);
+        lua_pushstring(L,"imaginary roots!");
+        return 2;
+    } else {
+        abc = sqrt(abc);
+        a = 2*a;
+        lua_pushnumber(L,(-b + abc)/a);
+        lua_pushnumber(L,(+b - abc)/a);
+        return 2;
+    }
+}
+
+static const luaL_reg mylib[] = {
+    {"createtable",l_createtable},
+    {"solve",l_solve},
+    {NULL,NULL}
+};
+
+int luaopen_mylib(lua_State *L)
+{
+    luaL_register (L, "mylib", mylib);
+    return 1;
+}
diff --git a/tests/md-test/config.ld b/tests/md-test/config.ld
new file mode 100644
index 0000000..1bb9271
--- /dev/null
+++ b/tests/md-test/config.ld
@@ -0,0 +1,3 @@
+project = 'md-test'
+title = 'Markdown Docs'
+format = 'markdown'
diff --git a/tests/md-test/mod2.lua b/tests/md-test/mod2.lua
new file mode 100644
index 0000000..4e73671
--- /dev/null
+++ b/tests/md-test/mod2.lua
@@ -0,0 +1,25 @@
+---------------------
+-- Another test module.
+-- This one uses _Markdown_ formating, and
+-- so can include goodies such as `code`
+-- and lists:
+--
+--  - one
+--  - two
+--  - three
+--
+-- @module mod2
+
+--- really basic function. Can contain links such as
+-- [this](http://lua-users.org/wiki/FindPage)
+-- @param first like `gsb(x)`
+-- @param second **bold** maybe? It can continue:
+--
+--  - another point
+--  - finish the damn list
+-- @param third as before
+function mod2.basic(first,second,third)
+
+end
+
+
diff --git a/tests/simple/simple.lua b/tests/simple/simple.lua
new file mode 100644
index 0000000..b05b8bb
--- /dev/null
+++ b/tests/simple/simple.lua
@@ -0,0 +1,12 @@
+------------
+-- A little old-style module
+local io = io
+-- we'll look for this
+module 'simple'
+
+-- if it were 'module (...)' then the name has to be deduced.
+
+--- return the answer.
+function answer()
+  return 42
+end
diff --git a/tools.lua b/tools.lua
new file mode 100644
index 0000000..70d6a58
--- /dev/null
+++ b/tools.lua
@@ -0,0 +1,265 @@
+---------
+-- General utility functions for ldoc
+-- @module tools
+
+require 'pl'
+local tools = {}
+local M = tools
+local append = table.insert
+local lexer = require 'lexer'
+local quit = utils.quit
+
+-- this constructs an iterator over a list of objects which returns only
+-- those objects where a field has a certain value. It's used to iterate
+-- only over functions or tables, etc.
+-- (something rather similar exists in LuaDoc)
+function M.type_iterator (list,field,value)
+    return function()
+        local i = 1
+        return function()
+            local val = list[i]
+            while val and val[field] ~= value do
+                i = i + 1
+                val = list[i]
+            end
+            i = i + 1
+            if val then return val end
+        end
+    end
+end
+
+-- KindMap is used to iterate over a set of categories, called _kinds_,
+-- and the associated iterator over all items in that category.
+-- For instance, a module contains functions, tables, etc and we will
+-- want to iterate over these categories in a specified order:
+--
+--  for kind, items in module.kinds() do
+--    print('kind',kind)
+--    for item in items() do print(item.name) end
+--  end
+--
+-- The kind is typically used as a label or a Title, so for type 'function' the
+-- kind is 'Functions' and so on.
+
+local KindMap = class()
+M.KindMap = KindMap
+
+-- calling a KindMap returns an iterator. This returns the kind, the iterator
+-- over the items of that type, and the corresponding type.
+function KindMap:__call ()
+    local i = 1
+    local klass = self.klass
+    return function()
+        local kind = klass.kinds[i]
+        if not kind then return nil end -- no more kinds
+        while not self[kind] do
+            i = i + 1
+            kind = klass.kinds[i]
+            if not kind then return nil end
+        end
+        i = i + 1
+        return kind, self[kind], klass.types_by_kind[kind]
+    end
+end
+
+-- called for each new item. It does not actually create separate lists,
+-- (although that would not break the interface) but creates iterators
+-- for that item type if not already created.
+function KindMap:add (item,items)
+    local kname = self.klass.types_by_tag[item.type]
+    if not self[kname] then
+        self[kname] = M.type_iterator (items,'type',item.type)
+    end
+end
+
+-- KindMap has a 'class constructor' which is used to modify
+-- any new base class.
+function KindMap._class_init (klass)
+    klass.kinds = {} -- list in correct order of kinds
+    klass.types_by_tag = {} -- indexed by tag
+    klass.types_by_kind = {} -- indexed by kind
+end
+
+
+function KindMap.add_kind (klass,tag,kind,subnames)
+    klass.types_by_tag[tag] = kind
+    klass.types_by_kind[kind] = {type=tag,subnames=subnames}
+    append(klass.kinds,kind)
+end
+
+
+----- some useful utility functions ------
+
+function M.module_basepath()
+    local lpath = List.split(package.path,';')
+    for p in lpath:iter() do
+        local p = path.dirname(p)
+        if path.isabs(p) then
+            return p
+        end
+    end
+end
+
+-- split a qualified name into the module part and the name part,
+-- e.g 'pl.utils.split' becomes 'pl.utils' and 'split'
+function M.split_dotted_name (s)
+    local s1,s2 = path.splitext(s)
+    if s2=='' then return nil
+    else  return s1,s2:sub(2)
+    end
+end
+
+-- expand lists of possibly qualified identifiers
+-- given something like {'one , two.2','three.drei.drie)'}
+-- it will output {"one","two.2","three.drei.drie"}
+function M.expand_comma_list (ls)
+    local new_ls = List()
+    for s in ls:iter() do
+        s = s:gsub('[^%.:%w]*$','')
+        if s:find ',' then
+            new_ls:extend(List.split(s,'%s*,%s*'))
+        else
+            new_ls:append(s)
+        end
+    end
+    return new_ls
+end
+
+function M.extract_identifier (value)
+    return value:match('([%.:_%w]+)')
+end
+
+function M.strip (s)
+    return s:gsub('^%s+',''):gsub('%s+$','')
+end
+
+function M.check_directory(d)
+    if not path.isdir(d) then
+        lfs.mkdir(d)
+    end
+end
+
+function M.check_file (f,original)
+    if not path.exists(f) then
+        dir.copyfile(original,f)
+    end
+end
+
+function M.writefile(name,text)
+    local ok,err = utils.writefile(name,text)
+    if err then quit(err) end
+end
+
+function M.name_of (lpath)
+    lpath,ext = path.splitext(lpath)
+    return lpath
+end
+
+function M.this_module_name (basename,fname)
+    local ext
+    if basename == '' then
+        --quit("module(...) needs package basename")
+        return M.name_of(fname)
+    end
+    basename = path.abspath(basename)
+    if basename:sub(-1,-1) ~= path.sep then
+        basename = basename..path.sep
+    end
+    local lpath,cnt = fname:gsub('^'..utils.escape(basename),'')
+    if cnt ~= 1 then quit("module(...) name deduction failed: base "..basename.." "..fname) end
+    lpath = lpath:gsub(path.sep,'.')
+    return M.name_of(lpath)
+end
+
+
+--------- lexer tools -----
+
+local tnext = lexer.skipws
+
+local function type_of (tok) return tok[1] end
+local function value_of (tok) return tok[2] end
+
+-- This parses Lua formal argument lists. It will return a list of argument
+-- names, which also has a comments field, which will contain any commments
+-- following the arguments. ldoc will use these in addition to explicit
+-- param tags.
+
+function M.get_parameters (tok)
+    local args = List()
+    args.comments = {}
+    local ltl = lexer.get_separated_list(tok)
+
+    if #ltl[1] == 0 then return args end -- no arguments
+
+    local function set_comment (idx,tok)
+        args.comments[args[idx]] = value_of(tok)
+    end
+
+    for i = 1,#ltl do
+        local tl = ltl[i]
+        if type_of(tl[1]) == 'comment' then
+            if i > 1 then set_comment(i-1,tl[1]) end
+            if #tl > 1 then
+                args:append(value_of(tl[2]))
+            end
+        else
+            args:append(value_of(tl[1]))
+        end
+        if i == #ltl then
+            local last_tok = tl[#tl]
+            if #tl > 1 and type_of(last_tok) == 'comment' then
+                set_comment(i,last_tok)
+            end
+        end
+    end
+
+    return args
+end
+
+-- parse a Lua identifier - contains names separated by . and :.
+function M.get_fun_name (tok)
+    local res = {}
+    local _,name = tnext(tok)
+    _,sep = tnext(tok)
+    while sep == '.' or sep == ':' do
+        append(res,name)
+        append(res,sep)
+        _,name = tnext(tok)
+        _,sep = tnext(tok)
+    end
+    append(res,name)
+    return table.concat(res)
+end
+
+-- space-skipping version of token iterator
+function M.space_skip_getter(tok)
+    return function ()
+        local t,v = tok()
+        while t and t == 'space' do
+            t,v = tok()
+        end
+        return t,v
+    end
+end
+
+-- an embarassing function. The PL Lua lexer does not do block comments
+-- when used in line-grabbing mode, and in fact (0.9.4) does not even
+-- do them properly in full-text mode, due to a ordering mistake.
+-- So, we do what we can ;)
+function M.grab_block_comment (v,tok,end1,end2)
+    local res = {v}
+    local t,last_v
+    repeat
+        last_v = v
+        t,v = tok()
+        append(res,v)
+    until last_v == end1 and v == end2
+    table.remove(res)
+    table.remove(res)
+    res = table.concat(res)
+    return 'comment',res
+end
+
+
+
+return tools