510 lines
18 KiB
Python
510 lines
18 KiB
Python
"""
|
|
Attempt to generate templates for module reference with Sphinx
|
|
|
|
To include extension modules, first identify them as valid in the
|
|
``_uri2path`` method, then handle them in the ``_parse_module_with_import``
|
|
script.
|
|
|
|
Notes
|
|
-----
|
|
This parsing is based on import and introspection of modules.
|
|
Previously functions and classes were found by parsing the text of .py files.
|
|
|
|
Extension modules should be discovered and included as well.
|
|
|
|
This is a modified version of a script originally shipped with the PyMVPA
|
|
project, then adapted for use first in NIPY and then in skimage. PyMVPA
|
|
is an MIT-licensed project.
|
|
"""
|
|
|
|
# Stdlib imports
|
|
import os
|
|
import re
|
|
|
|
from types import BuiltinFunctionType, FunctionType, ModuleType
|
|
|
|
# suppress print statements (warnings for empty files)
|
|
DEBUG = True
|
|
|
|
|
|
class ApiDocWriter(object):
|
|
''' Class for automatic detection and parsing of API docs
|
|
to Sphinx-parsable reST format'''
|
|
|
|
# only separating first two levels
|
|
rst_section_levels = ['*', '=', '-', '~', '^']
|
|
|
|
def __init__(self,
|
|
package_name,
|
|
rst_extension='.rst',
|
|
package_skip_patterns=None,
|
|
module_skip_patterns=None,
|
|
):
|
|
r''' Initialize package for parsing
|
|
|
|
Parameters
|
|
----------
|
|
package_name : string
|
|
Name of the top-level package. *package_name* must be the
|
|
name of an importable package
|
|
rst_extension : string, optional
|
|
Extension for reST files, default '.rst'
|
|
package_skip_patterns : None or sequence of {strings, regexps}
|
|
Sequence of strings giving URIs of packages to be excluded
|
|
Operates on the package path, starting at (including) the
|
|
first dot in the package path, after *package_name* - so,
|
|
if *package_name* is ``sphinx``, then ``sphinx.util`` will
|
|
result in ``.util`` being passed for earching by these
|
|
regexps. If is None, gives default. Default is:
|
|
['\.tests$']
|
|
module_skip_patterns : None or sequence
|
|
Sequence of strings giving URIs of modules to be excluded
|
|
Operates on the module name including preceding URI path,
|
|
back to the first dot after *package_name*. For example
|
|
``sphinx.util.console`` results in the string to search of
|
|
``.util.console``
|
|
If is None, gives default. Default is:
|
|
['\.setup$', '\._']
|
|
'''
|
|
if package_skip_patterns is None:
|
|
package_skip_patterns = ['\\.tests$']
|
|
if module_skip_patterns is None:
|
|
module_skip_patterns = ['\\.setup$', '\\._']
|
|
self.package_name = package_name
|
|
self.rst_extension = rst_extension
|
|
self.package_skip_patterns = package_skip_patterns
|
|
self.module_skip_patterns = module_skip_patterns
|
|
|
|
def get_package_name(self):
|
|
return self._package_name
|
|
|
|
def set_package_name(self, package_name):
|
|
''' Set package_name
|
|
|
|
>>> docwriter = ApiDocWriter('sphinx')
|
|
>>> import sphinx
|
|
>>> docwriter.root_path == sphinx.__path__[0]
|
|
True
|
|
>>> docwriter.package_name = 'docutils'
|
|
>>> import docutils
|
|
>>> docwriter.root_path == docutils.__path__[0]
|
|
True
|
|
'''
|
|
# It's also possible to imagine caching the module parsing here
|
|
self._package_name = package_name
|
|
root_module = self._import(package_name)
|
|
self.root_path = root_module.__path__[-1]
|
|
self.written_modules = None
|
|
|
|
package_name = property(get_package_name, set_package_name, None,
|
|
'get/set package_name')
|
|
|
|
def _import(self, name):
|
|
''' Import namespace package '''
|
|
mod = __import__(name)
|
|
components = name.split('.')
|
|
for comp in components[1:]:
|
|
mod = getattr(mod, comp)
|
|
return mod
|
|
|
|
def _get_object_name(self, line):
|
|
''' Get second token in line
|
|
>>> docwriter = ApiDocWriter('sphinx')
|
|
>>> docwriter._get_object_name(" def func(): ")
|
|
'func'
|
|
>>> docwriter._get_object_name(" class Klass(object): ")
|
|
'Klass'
|
|
>>> docwriter._get_object_name(" class Klass: ")
|
|
'Klass'
|
|
'''
|
|
name = line.split()[1].split('(')[0].strip()
|
|
# in case we have classes which are not derived from object
|
|
# ie. old style classes
|
|
return name.rstrip(':')
|
|
|
|
def _uri2path(self, uri):
|
|
''' Convert uri to absolute filepath
|
|
|
|
Parameters
|
|
----------
|
|
uri : string
|
|
URI of python module to return path for
|
|
|
|
Returns
|
|
-------
|
|
path : None or string
|
|
Returns None if there is no valid path for this URI
|
|
Otherwise returns absolute file system path for URI
|
|
|
|
Examples
|
|
--------
|
|
>>> docwriter = ApiDocWriter('sphinx')
|
|
>>> import sphinx
|
|
>>> modpath = sphinx.__path__[0]
|
|
>>> res = docwriter._uri2path('sphinx.builder')
|
|
>>> res == os.path.join(modpath, 'builder.py')
|
|
True
|
|
>>> res = docwriter._uri2path('sphinx')
|
|
>>> res == os.path.join(modpath, '__init__.py')
|
|
True
|
|
>>> docwriter._uri2path('sphinx.does_not_exist')
|
|
|
|
'''
|
|
if uri == self.package_name:
|
|
return os.path.join(self.root_path, '__init__.py')
|
|
path = uri.replace(self.package_name + '.', '')
|
|
path = path.replace('.', os.path.sep)
|
|
path = os.path.join(self.root_path, path)
|
|
# XXX maybe check for extensions as well?
|
|
if os.path.exists(path + '.py'): # file
|
|
path += '.py'
|
|
elif os.path.exists(os.path.join(path, '__init__.py')):
|
|
path = os.path.join(path, '__init__.py')
|
|
else:
|
|
return None
|
|
return path
|
|
|
|
def _path2uri(self, dirpath):
|
|
''' Convert directory path to uri '''
|
|
package_dir = self.package_name.replace('.', os.path.sep)
|
|
relpath = dirpath.replace(self.root_path, package_dir)
|
|
if relpath.startswith(os.path.sep):
|
|
relpath = relpath[1:]
|
|
return relpath.replace(os.path.sep, '.')
|
|
|
|
def _parse_module(self, uri):
|
|
''' Parse module defined in *uri* '''
|
|
filename = self._uri2path(uri)
|
|
if filename is None:
|
|
print(filename, 'erk')
|
|
# nothing that we could handle here.
|
|
return ([],[])
|
|
f = open(filename, 'rt')
|
|
functions, classes = self._parse_lines(f)
|
|
f.close()
|
|
return functions, classes
|
|
|
|
def _parse_module_with_import(self, uri):
|
|
"""Look for functions and classes in an importable module.
|
|
|
|
Parameters
|
|
----------
|
|
uri : str
|
|
The name of the module to be parsed. This module needs to be
|
|
importable.
|
|
|
|
Returns
|
|
-------
|
|
functions : list of str
|
|
A list of (public) function names in the module.
|
|
classes : list of str
|
|
A list of (public) class names in the module.
|
|
submodules : list of str
|
|
A list of (public) submodule names in the module.
|
|
"""
|
|
mod = __import__(uri, fromlist=[uri.split('.')[-1]])
|
|
# find all public objects in the module.
|
|
obj_strs = getattr(mod, '__all__',
|
|
[obj for obj in dir(mod) if not obj.startswith('_')])
|
|
functions = []
|
|
classes = []
|
|
submodules = []
|
|
for obj_str in obj_strs:
|
|
# find the actual object from its string representation
|
|
if obj_str not in mod.__dict__:
|
|
continue
|
|
obj = mod.__dict__[obj_str]
|
|
|
|
# figure out if obj is a function or class
|
|
if isinstance(obj, (FunctionType, BuiltinFunctionType)):
|
|
functions.append(obj_str)
|
|
elif isinstance(obj, ModuleType) and 'skimage' in mod.__name__:
|
|
submodules.append(obj_str)
|
|
else:
|
|
try:
|
|
issubclass(obj, object)
|
|
classes.append(obj_str)
|
|
except TypeError:
|
|
# not a function or class
|
|
pass
|
|
return functions, classes, submodules
|
|
|
|
def _parse_lines(self, linesource):
|
|
''' Parse lines of text for functions and classes '''
|
|
functions = []
|
|
classes = []
|
|
for line in linesource:
|
|
if line.startswith('def ') and line.count('('):
|
|
# exclude private stuff
|
|
name = self._get_object_name(line)
|
|
if not name.startswith('_'):
|
|
functions.append(name)
|
|
elif line.startswith('class '):
|
|
# exclude private stuff
|
|
name = self._get_object_name(line)
|
|
if not name.startswith('_'):
|
|
classes.append(name)
|
|
else:
|
|
pass
|
|
functions.sort()
|
|
classes.sort()
|
|
return functions, classes
|
|
|
|
def generate_api_doc(self, uri):
|
|
'''Make autodoc documentation template string for a module
|
|
|
|
Parameters
|
|
----------
|
|
uri : string
|
|
python location of module - e.g 'sphinx.builder'
|
|
|
|
Returns
|
|
-------
|
|
S : string
|
|
Contents of API doc
|
|
'''
|
|
# get the names of all classes and functions
|
|
functions, classes, submodules = self._parse_module_with_import(uri)
|
|
if not (len(functions) or len(classes) or len(submodules)) and DEBUG:
|
|
print('WARNING: Empty -', uri)
|
|
return ''
|
|
functions = sorted(functions)
|
|
classes = sorted(classes)
|
|
submodules = sorted(submodules)
|
|
|
|
# Make a shorter version of the uri that omits the package name for
|
|
# titles
|
|
uri_short = re.sub(r'^%s\.' % self.package_name,'',uri)
|
|
|
|
ad = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n'
|
|
|
|
# Set the chapter title to read 'module' for all modules except for the
|
|
# main packages
|
|
if '.' in uri:
|
|
title = 'Module: :mod:`' + uri_short + '`'
|
|
else:
|
|
title = ':mod:`' + uri_short + '`'
|
|
ad += title + '\n' + self.rst_section_levels[1] * len(title)
|
|
|
|
ad += '\n.. automodule:: ' + uri + '\n'
|
|
ad += '\n.. currentmodule:: ' + uri + '\n'
|
|
ad += '.. autosummary::\n\n'
|
|
for f in functions:
|
|
ad += ' ' + uri + '.' + f + '\n'
|
|
ad += '\n'
|
|
for c in classes:
|
|
ad += ' ' + uri + '.' + c + '\n'
|
|
ad += '\n'
|
|
for m in submodules:
|
|
ad += ' ' + uri + '.' + m + '\n'
|
|
ad += '\n'
|
|
|
|
for f in functions:
|
|
# must NOT exclude from index to keep cross-refs working
|
|
full_f = uri + '.' + f
|
|
ad += f + '\n'
|
|
ad += self.rst_section_levels[2] * len(f) + '\n'
|
|
ad += '\n.. autofunction:: ' + full_f + '\n\n'
|
|
ad += '\n.. include:: ' + full_f + '.examples\n\n'
|
|
for c in classes:
|
|
ad += '\n:class:`' + c + '`\n' \
|
|
+ self.rst_section_levels[2] * \
|
|
(len(c)+9) + '\n\n'
|
|
ad += '\n.. autoclass:: ' + c + '\n'
|
|
# must NOT exclude from index to keep cross-refs working
|
|
ad += ' :members:\n' \
|
|
' :undoc-members:\n' \
|
|
' :show-inheritance:\n' \
|
|
'\n' \
|
|
' .. automethod:: __init__\n'
|
|
full_c = uri + '.' + c
|
|
ad += '\n.. include:: ' + full_c + '.examples\n\n'
|
|
return ad
|
|
|
|
def _survives_exclude(self, matchstr, match_type):
|
|
''' Returns True if *matchstr* does not match patterns
|
|
|
|
``self.package_name`` removed from front of string if present
|
|
|
|
Examples
|
|
--------
|
|
>>> dw = ApiDocWriter('sphinx')
|
|
>>> dw._survives_exclude('sphinx.okpkg', 'package')
|
|
True
|
|
>>> dw.package_skip_patterns.append('^\\.badpkg$')
|
|
>>> dw._survives_exclude('sphinx.badpkg', 'package')
|
|
False
|
|
>>> dw._survives_exclude('sphinx.badpkg', 'module')
|
|
True
|
|
>>> dw._survives_exclude('sphinx.badmod', 'module')
|
|
True
|
|
>>> dw.module_skip_patterns.append('^\\.badmod$')
|
|
>>> dw._survives_exclude('sphinx.badmod', 'module')
|
|
False
|
|
'''
|
|
if match_type == 'module':
|
|
patterns = self.module_skip_patterns
|
|
elif match_type == 'package':
|
|
patterns = self.package_skip_patterns
|
|
else:
|
|
raise ValueError('Cannot interpret match type "%s"'
|
|
% match_type)
|
|
# Match to URI without package name
|
|
L = len(self.package_name)
|
|
if matchstr[:L] == self.package_name:
|
|
matchstr = matchstr[L:]
|
|
for pat in patterns:
|
|
try:
|
|
pat.search
|
|
except AttributeError:
|
|
pat = re.compile(pat)
|
|
if pat.search(matchstr):
|
|
return False
|
|
return True
|
|
|
|
def discover_modules(self):
|
|
r''' Return module sequence discovered from ``self.package_name``
|
|
|
|
|
|
Parameters
|
|
----------
|
|
None
|
|
|
|
Returns
|
|
-------
|
|
mods : sequence
|
|
Sequence of module names within ``self.package_name``
|
|
|
|
Examples
|
|
--------
|
|
>>> dw = ApiDocWriter('sphinx')
|
|
>>> mods = dw.discover_modules()
|
|
>>> 'sphinx.util' in mods
|
|
True
|
|
>>> dw.package_skip_patterns.append('\.util$')
|
|
>>> 'sphinx.util' in dw.discover_modules()
|
|
False
|
|
>>>
|
|
'''
|
|
modules = [self.package_name]
|
|
# raw directory parsing
|
|
for dirpath, dirnames, filenames in os.walk(self.root_path):
|
|
# Check directory names for packages
|
|
root_uri = self._path2uri(os.path.join(self.root_path,
|
|
dirpath))
|
|
for dirname in dirnames[:]: # copy list - we modify inplace
|
|
package_uri = '.'.join((root_uri, dirname))
|
|
if (self._uri2path(package_uri) and
|
|
self._survives_exclude(package_uri, 'package')):
|
|
modules.append(package_uri)
|
|
else:
|
|
dirnames.remove(dirname)
|
|
return sorted(modules)
|
|
|
|
def write_modules_api(self, modules, outdir):
|
|
# write the list
|
|
written_modules = []
|
|
public_modules = [m for m in modules
|
|
if not m.split('.')[-1].startswith('_')]
|
|
for m in public_modules:
|
|
api_str = self.generate_api_doc(m)
|
|
if not api_str:
|
|
continue
|
|
# write out to file
|
|
outfile = os.path.join(outdir,
|
|
m + self.rst_extension)
|
|
fileobj = open(outfile, 'wt')
|
|
fileobj.write(api_str)
|
|
fileobj.close()
|
|
written_modules.append(m)
|
|
self.written_modules = written_modules
|
|
|
|
def write_api_docs(self, outdir):
|
|
"""Generate API reST files.
|
|
|
|
Parameters
|
|
----------
|
|
outdir : string
|
|
Directory name in which to store files
|
|
We create automatic filenames for each module
|
|
|
|
Returns
|
|
-------
|
|
None
|
|
|
|
Notes
|
|
-----
|
|
Sets self.written_modules to list of written modules
|
|
"""
|
|
if not os.path.exists(outdir):
|
|
os.mkdir(outdir)
|
|
# compose list of modules
|
|
modules = self.discover_modules()
|
|
self.write_modules_api(modules, outdir)
|
|
|
|
def write_index(self, outdir, froot='gen', relative_to=None):
|
|
"""Make a reST API index file from written files
|
|
|
|
Parameters
|
|
----------
|
|
path : string
|
|
Filename to write index to
|
|
outdir : string
|
|
Directory to which to write generated index file
|
|
froot : string, optional
|
|
root (filename without extension) of filename to write to
|
|
Defaults to 'gen'. We add ``self.rst_extension``.
|
|
relative_to : string
|
|
path to which written filenames are relative. This
|
|
component of the written file path will be removed from
|
|
outdir, in the generated index. Default is None, meaning,
|
|
leave path as it is.
|
|
"""
|
|
if self.written_modules is None:
|
|
raise ValueError('No modules written')
|
|
# Get full filename path
|
|
path = os.path.join(outdir, froot+self.rst_extension)
|
|
# Path written into index is relative to rootpath
|
|
if relative_to is not None:
|
|
relpath = (outdir + os.path.sep).replace(relative_to + os.path.sep, '')
|
|
else:
|
|
relpath = outdir
|
|
print("outdir: ", relpath)
|
|
idx = open(path,'wt')
|
|
w = idx.write
|
|
w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n')
|
|
|
|
# We look at the module name. If it is `skimage`, display, if `skimage.submodule`, only show `submodule`,
|
|
# if it is `skimage.submodule.subsubmodule`, ignore.
|
|
|
|
title = "API Reference for skimage |version|"
|
|
w(title + "\n")
|
|
w("=" * len(title) + "\n\n")
|
|
|
|
subtitle = "Submodules"
|
|
w(subtitle + "\n")
|
|
w("-" * len(subtitle) + "\n\n")
|
|
|
|
for f in self.written_modules:
|
|
module_name = f.split('.')
|
|
if len(module_name) > 2:
|
|
continue
|
|
elif len(module_name) == 1:
|
|
module_name = module_name[0]
|
|
prefix = "-"
|
|
elif len(module_name) == 2:
|
|
module_name = module_name[1]
|
|
prefix = "\n -"
|
|
w('{0} `{1} <{2}.html>`__\n'.format(prefix, module_name, os.path.join(f)))
|
|
w('\n')
|
|
|
|
subtitle = "Submodule Contents"
|
|
w(subtitle + "\n")
|
|
w("-" * len(subtitle) + "\n\n")
|
|
|
|
w('.. toctree::\n')
|
|
w(' :maxdepth: 2\n\n')
|
|
for f in self.written_modules:
|
|
w(' %s\n' % os.path.join(relpath,f))
|
|
idx.close()
|