627 lines
21 KiB
Python
627 lines
21 KiB
Python
"""
|
|
Example generation from python files.
|
|
|
|
Generate the rst files for the examples by iterating over the python
|
|
example files. Files that generate images should start with 'plot'.
|
|
|
|
To generate your own examples, add this extension to the list of
|
|
``extensions``in your Sphinx configuration file. In addition, make sure the
|
|
example directory(ies) in `plot2rst_paths` (see below) points to a directory
|
|
with examples named `plot_*.py` and include an `index.rst` file.
|
|
|
|
This code was adapted from scikit-image, which took it from scikit-learn.
|
|
|
|
Options
|
|
-------
|
|
The ``plot2rst`` extension accepts the following options:
|
|
|
|
plot2rst_paths : length-2 tuple, or list of tuples
|
|
Tuple or list of tuples of paths to (python plot, generated rst) files,
|
|
i.e. (source, destination). Note that both paths are relative to Sphinx
|
|
'source' directory. Defaults to ('../examples', 'auto_examples')
|
|
|
|
plot2rst_rcparams : dict
|
|
Matplotlib configuration parameters. See
|
|
https://matplotlib.org/tutorials/introductory/customizing.html for details.
|
|
|
|
plot2rst_default_thumb : str
|
|
Path (relative to doc root) of default thumbnail image.
|
|
|
|
plot2rst_thumb_shape : float
|
|
Shape of thumbnail in pixels. The image is resized to fit within this shape
|
|
and the excess is filled with white pixels. This fixed size ensures that
|
|
that gallery images are displayed in a grid.
|
|
|
|
plot2rst_plot_tag : str
|
|
When this tag is found in the example file, the current plot is saved and
|
|
tag is replaced with plot path. Defaults to 'PLOT2RST.current_figure'.
|
|
|
|
|
|
Suggested CSS definitions
|
|
-------------------------
|
|
|
|
div.body h2 {
|
|
border-bottom: 1px solid #BBB;
|
|
clear: left;
|
|
}
|
|
|
|
/*---- example gallery ----*/
|
|
|
|
.gallery.figure {
|
|
float: left;
|
|
margin: 1em;
|
|
}
|
|
|
|
.gallery.figure img{
|
|
display: block;
|
|
margin-left: auto;
|
|
margin-right: auto;
|
|
width: 200px;
|
|
}
|
|
|
|
.gallery.figure .caption {
|
|
width: 200px;
|
|
text-align: center !important;
|
|
}
|
|
|
|
"""
|
|
import os
|
|
import re
|
|
import shutil
|
|
import token
|
|
import tokenize
|
|
import traceback
|
|
import itertools
|
|
|
|
import numpy as np
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
|
|
from skimage import io
|
|
from skimage import transform
|
|
from skimage.util.dtype import dtype_range
|
|
|
|
from notebook_doc import Notebook
|
|
|
|
from docutils.core import publish_parts
|
|
from sphinx.domains.python import PythonDomain
|
|
|
|
LITERALINCLUDE = """
|
|
.. literalinclude:: {src_name}
|
|
:lines: {code_start}-
|
|
|
|
"""
|
|
|
|
CODE_LINK = """
|
|
|
|
**Python source code:** :download:`download <{0}>`
|
|
(generated using ``skimage`` |version|)
|
|
|
|
"""
|
|
|
|
NOTEBOOK_LINK = """
|
|
|
|
**IPython Notebook:** :download:`download <{0}>`
|
|
(generated using ``skimage`` |version|)
|
|
|
|
"""
|
|
|
|
TOCTREE_TEMPLATE = """
|
|
.. toctree::
|
|
:hidden:
|
|
|
|
%s
|
|
|
|
"""
|
|
|
|
IMAGE_TEMPLATE = """
|
|
.. image:: images/%s
|
|
:align: center
|
|
|
|
"""
|
|
|
|
GALLERY_IMAGE_TEMPLATE = """
|
|
.. figure:: %(thumb)s
|
|
:figclass: gallery
|
|
:target: ./%(source)s.html
|
|
|
|
:ref:`example_%(link_name)s`
|
|
|
|
"""
|
|
|
|
|
|
class Path(str):
|
|
"""Path object for manipulating directory and file paths."""
|
|
|
|
def __new__(self, path):
|
|
return str.__new__(self, path)
|
|
|
|
@property
|
|
def isdir(self):
|
|
return os.path.isdir(self)
|
|
|
|
@property
|
|
def exists(self):
|
|
"""Return True if path exists"""
|
|
return os.path.exists(self)
|
|
|
|
def pjoin(self, *args):
|
|
"""Join paths. `p` prefix prevents confusion with string method."""
|
|
return self.__class__(os.path.join(self, *args))
|
|
|
|
def psplit(self):
|
|
"""Split paths. `p` prefix prevents confusion with string method."""
|
|
return [self.__class__(p) for p in os.path.split(self)]
|
|
|
|
def makedirs(self):
|
|
if not self.exists:
|
|
os.makedirs(self)
|
|
|
|
def listdir(self):
|
|
return os.listdir(self)
|
|
|
|
def format(self, *args, **kwargs):
|
|
return self.__class__(super(Path, self).format(*args, **kwargs))
|
|
|
|
def __add__(self, other):
|
|
return self.__class__(super(Path, self).__add__(other))
|
|
|
|
def __iadd__(self, other):
|
|
return self.__add__(other)
|
|
|
|
|
|
def setup(app):
|
|
app.connect('builder-inited', generate_example_galleries)
|
|
|
|
app.add_config_value('plot2rst_paths',
|
|
('../examples', 'auto_examples'), True)
|
|
app.add_config_value('plot2rst_rcparams', {}, True)
|
|
app.add_config_value('plot2rst_default_thumb', None, True)
|
|
app.add_config_value('plot2rst_thumb_shape', (250, 300), True)
|
|
app.add_config_value('plot2rst_plot_tag', 'PLOT2RST.current_figure', True)
|
|
app.add_config_value('plot2rst_index_name', 'index', True)
|
|
|
|
|
|
def generate_example_galleries(app):
|
|
cfg = app.builder.config
|
|
|
|
if isinstance(cfg.source_suffix, list):
|
|
cfg.source_suffix_str = cfg.source_suffix[0]
|
|
else:
|
|
cfg.source_suffix_str = cfg.source_suffix
|
|
|
|
doc_src = Path(os.path.abspath(app.builder.srcdir)) # path/to/doc/source
|
|
|
|
if isinstance(cfg.plot2rst_paths, tuple):
|
|
cfg.plot2rst_paths = [cfg.plot2rst_paths]
|
|
for src_dest in cfg.plot2rst_paths:
|
|
plot_path, rst_path = [Path(p) for p in src_dest]
|
|
example_dir = doc_src.pjoin(plot_path)
|
|
rst_dir = doc_src.pjoin(rst_path)
|
|
generate_examples_and_gallery(example_dir, rst_dir, cfg)
|
|
|
|
|
|
def generate_examples_and_gallery(example_dir, rst_dir, cfg):
|
|
"""Generate rst from examples and create gallery to showcase examples."""
|
|
if not example_dir.exists:
|
|
print("No example directory found at", example_dir)
|
|
return
|
|
rst_dir.makedirs()
|
|
|
|
# we create an index.rst with all examples
|
|
with open(rst_dir.pjoin('index'+cfg.source_suffix_str), 'w') as gallery_index:
|
|
# Here we don't use an os.walk, but we recurse only twice: flat is
|
|
# better than nested.
|
|
write_gallery(gallery_index, example_dir, rst_dir, cfg)
|
|
for d in sorted(example_dir.listdir()):
|
|
example_sub = example_dir.pjoin(d)
|
|
if example_sub.isdir:
|
|
rst_sub = rst_dir.pjoin(d)
|
|
rst_sub.makedirs()
|
|
write_gallery(gallery_index, example_sub, rst_sub, cfg, depth=1)
|
|
gallery_index.flush()
|
|
|
|
|
|
def write_gallery(gallery_index, src_dir, rst_dir, cfg, depth=0):
|
|
"""Generate the rst files for an example directory, i.e. gallery.
|
|
|
|
Write rst files from python examples and add example links to gallery.
|
|
|
|
Parameters
|
|
----------
|
|
gallery_index : file
|
|
Index file for plot gallery.
|
|
src_dir : 'str'
|
|
Source directory for python examples.
|
|
rst_dir : 'str'
|
|
Destination directory for rst files generated from python examples.
|
|
cfg : config object
|
|
Sphinx config object created by Sphinx.
|
|
"""
|
|
index_name = cfg.plot2rst_index_name + cfg.source_suffix_str
|
|
gallery_template = src_dir.pjoin(index_name)
|
|
if not os.path.exists(gallery_template):
|
|
print(src_dir)
|
|
print(80*'_')
|
|
print('Example directory %s does not have a %s file'
|
|
% (src_dir, index_name))
|
|
print('Skipping this directory')
|
|
print(80*'_')
|
|
return
|
|
|
|
with open(gallery_template) as f:
|
|
gallery_description = f.read()
|
|
gallery_index.write('\n\n%s\n\n' % gallery_description)
|
|
|
|
rst_dir.makedirs()
|
|
examples = [fname for fname in sorted(src_dir.listdir(), key=_plots_first)
|
|
if fname.endswith('py')]
|
|
ex_names = [ex[:-3] for ex in examples] # strip '.py' extension
|
|
if depth == 0:
|
|
sub_dir = Path('')
|
|
else:
|
|
sub_dir_list = src_dir.psplit()[-depth:]
|
|
sub_dir = Path('/'.join(sub_dir_list) + '/')
|
|
joiner = '\n %s' % sub_dir
|
|
gallery_index.write(TOCTREE_TEMPLATE % (sub_dir + joiner.join(ex_names)))
|
|
|
|
for src_name in examples:
|
|
|
|
try:
|
|
write_example(src_name, src_dir, rst_dir, cfg)
|
|
except Exception:
|
|
print("Exception raised while running:")
|
|
print("%s in %s" % (src_name, src_dir))
|
|
print('~' * 60)
|
|
traceback.print_exc()
|
|
print('~' * 60)
|
|
continue
|
|
|
|
link_name = sub_dir.pjoin(src_name)
|
|
link_name = link_name.replace(os.path.sep, '_')
|
|
if link_name.startswith('._'):
|
|
link_name = link_name[2:]
|
|
|
|
info = {}
|
|
info['thumb'] = sub_dir.pjoin('images/thumb', src_name[:-3] + '.png')
|
|
info['source'] = sub_dir + src_name[:-3]
|
|
info['link_name'] = link_name
|
|
gallery_index.write(GALLERY_IMAGE_TEMPLATE % info)
|
|
|
|
|
|
def _plots_first(fname):
|
|
"""Decorate filename so that examples with plots are displayed first."""
|
|
if not (fname.startswith('plot') and fname.endswith('.py')):
|
|
return 'zz' + fname
|
|
return fname
|
|
|
|
|
|
def write_example(src_name, src_dir, rst_dir, cfg):
|
|
"""Write rst file from a given python example.
|
|
|
|
Parameters
|
|
----------
|
|
src_name : str
|
|
Name of example file.
|
|
src_dir : 'str'
|
|
Source directory for python examples.
|
|
rst_dir : 'str'
|
|
Destination directory for rst files generated from python examples.
|
|
cfg : config object
|
|
Sphinx config object created by Sphinx.
|
|
"""
|
|
last_dir = src_dir.psplit()[-1]
|
|
# to avoid leading . in file names, and wrong names in links
|
|
if last_dir == '.' or last_dir == 'examples':
|
|
last_dir = Path('')
|
|
else:
|
|
last_dir += '_'
|
|
|
|
src_path = src_dir.pjoin(src_name)
|
|
example_file = rst_dir.pjoin(src_name)
|
|
shutil.copyfile(src_path, example_file)
|
|
|
|
image_dir = rst_dir.pjoin('images')
|
|
thumb_dir = image_dir.pjoin('thumb')
|
|
notebook_dir = rst_dir.pjoin('notebook')
|
|
image_dir.makedirs()
|
|
thumb_dir.makedirs()
|
|
notebook_dir.makedirs()
|
|
|
|
base_image_name = os.path.splitext(src_name)[0]
|
|
image_path = image_dir.pjoin(base_image_name + '_{0}.png')
|
|
|
|
basename, py_ext = os.path.splitext(src_name)
|
|
|
|
rst_path = rst_dir.pjoin(basename + cfg.source_suffix_str)
|
|
notebook_path = notebook_dir.pjoin(basename + '.ipynb')
|
|
|
|
if _plots_are_current(src_path, image_path) and rst_path.exists and \
|
|
notebook_path.exists:
|
|
return
|
|
|
|
print('plot2rst: %s' % basename)
|
|
|
|
blocks = split_code_and_text_blocks(example_file)
|
|
if blocks[0][2].startswith('#!'):
|
|
blocks.pop(0) # don't add shebang line to rst file.
|
|
|
|
rst_link = '.. _example_%s:\n\n' % (last_dir + src_name)
|
|
figure_list, rst = process_blocks(blocks, src_path, image_path, cfg)
|
|
|
|
has_inline_plots = any(cfg.plot2rst_plot_tag in b[2] for b in blocks)
|
|
if has_inline_plots:
|
|
example_rst = ''.join([rst_link, rst])
|
|
else:
|
|
# print first block of text, display all plots, then display code.
|
|
first_text_block = [b for b in blocks if b[0] == 'text'][0]
|
|
label, (start, end), content = first_text_block
|
|
figure_list = save_all_figures(image_path)
|
|
rst_blocks = [IMAGE_TEMPLATE % f.lstrip('/') for f in figure_list]
|
|
|
|
example_rst = rst_link
|
|
example_rst += eval(content)
|
|
example_rst += ''.join(rst_blocks)
|
|
code_info = dict(src_name=src_name, code_start=end)
|
|
example_rst += LITERALINCLUDE.format(**code_info)
|
|
|
|
example_rst += CODE_LINK.format(src_name)
|
|
ipnotebook_name = src_name.replace('.py', '.ipynb')
|
|
ipnotebook_name = './notebook/' + ipnotebook_name
|
|
example_rst += NOTEBOOK_LINK.format(ipnotebook_name)
|
|
|
|
with open(rst_path, 'w') as f:
|
|
f.write(example_rst)
|
|
|
|
thumb_path = thumb_dir.pjoin(src_name[:-3] + '.png')
|
|
first_image_file = image_dir.pjoin(figure_list[0].lstrip('/'))
|
|
if first_image_file.exists:
|
|
first_image = io.imread(first_image_file)
|
|
save_thumbnail(first_image, thumb_path, cfg.plot2rst_thumb_shape)
|
|
|
|
if not thumb_path.exists:
|
|
if cfg.plot2rst_default_thumb is None:
|
|
print("WARNING: No plots found and default thumbnail not defined.")
|
|
print("Specify 'plot2rst_default_thumb' in Sphinx config file.")
|
|
else:
|
|
shutil.copy(cfg.plot2rst_default_thumb, thumb_path)
|
|
|
|
# Export example to IPython notebook
|
|
nb = Notebook()
|
|
|
|
# Add sphinx roles to the examples, otherwise docutils
|
|
# cannot compile the ReST for the notebook
|
|
sphinx_roles = PythonDomain.roles.keys()
|
|
preamble = '\n'.join('.. role:: py:{0}(literal)\n'.format(role)
|
|
for role in sphinx_roles)
|
|
|
|
# Grab all references to inject them in cells where needed
|
|
ref_regexp = re.compile('\n(\\.\\. \\[(\\d+)\\].*(?:\n[ ]{7,8}.*)+)')
|
|
math_role_regexp = re.compile(':math:`(.*?)`')
|
|
|
|
text = '\n'.join((content for (cell_type, _, content) in blocks
|
|
if cell_type != 'code'))
|
|
|
|
references = re.findall(ref_regexp, text)
|
|
|
|
for (cell_type, _, content) in blocks:
|
|
if cell_type == 'code':
|
|
nb.add_cell(content, cell_type='code')
|
|
else:
|
|
if content.startswith('r'):
|
|
content = content.replace('r"""', '')
|
|
escaped = False
|
|
else:
|
|
content = content.replace('"""', '')
|
|
escaped = True
|
|
|
|
if not escaped:
|
|
content = content.replace("\\", "\\\\")
|
|
|
|
content = content.replace('.. seealso::', '**See also:**')
|
|
content = re.sub(math_role_regexp, r'$\1$', content)
|
|
|
|
# Remove math directive when rendering notebooks
|
|
# until we implement a smarter way of capturing and replacing
|
|
# its content
|
|
content = content.replace('.. math::', '')
|
|
|
|
if not content.strip():
|
|
continue
|
|
|
|
content = (preamble + content).rstrip('\n')
|
|
content = '\n'.join([line for line in content.split('\n') if
|
|
not line.startswith('.. image')])
|
|
|
|
# Remove reference links until we can figure out a better way to
|
|
# preserve them
|
|
for (reference, ref_id) in references:
|
|
ref_tag = '[{0}]_'.format(ref_id)
|
|
if ref_tag in content:
|
|
content = content.replace(ref_tag, ref_tag[:-1])
|
|
|
|
html = publish_parts(content, writer_name='html')['html_body']
|
|
nb.add_cell(html, cell_type='markdown')
|
|
|
|
with open(notebook_path, 'w') as f:
|
|
f.write(nb.json())
|
|
|
|
|
|
def save_thumbnail(image, thumb_path, shape):
|
|
"""Save image as a thumbnail with the specified shape.
|
|
|
|
The image is first resized to fit within the specified shape and then
|
|
centered in an array of the specified shape before saving.
|
|
"""
|
|
rescale = min(float(w_1) / w_2 for w_1, w_2 in zip(shape, image.shape))
|
|
small_shape = (rescale * np.asarray(image.shape[:2])).astype(int)
|
|
small_image = transform.resize(image, small_shape)
|
|
|
|
if len(image.shape) == 3:
|
|
shape = shape + (image.shape[2],)
|
|
background_value = dtype_range[small_image.dtype.type][1]
|
|
thumb = background_value * np.ones(shape, dtype=small_image.dtype)
|
|
|
|
i = (shape[0] - small_shape[0]) // 2
|
|
j = (shape[1] - small_shape[1]) // 2
|
|
thumb[i:i+small_shape[0], j:j+small_shape[1]] = small_image
|
|
|
|
io.imsave(thumb_path, thumb)
|
|
|
|
|
|
def _plots_are_current(src_path, image_path):
|
|
first_image_file = Path(image_path.format(1))
|
|
needs_replot = (not first_image_file.exists or
|
|
_mod_time(first_image_file) <= _mod_time(src_path))
|
|
return not needs_replot
|
|
|
|
|
|
def _mod_time(file_path):
|
|
return os.stat(file_path).st_mtime
|
|
|
|
|
|
def split_code_and_text_blocks(source_file):
|
|
"""Return list with source file separated into code and text blocks.
|
|
|
|
Returns
|
|
-------
|
|
blocks : list of (label, (start, end+1), content)
|
|
List where each element is a tuple with the label ('text' or 'code'),
|
|
the (start, end+1) line numbers, and content string of block.
|
|
"""
|
|
block_edges, idx_first_text_block = get_block_edges(source_file)
|
|
|
|
with open(source_file) as f:
|
|
source_lines = f.readlines()
|
|
|
|
# Every other block should be a text block
|
|
idx_text_block = np.arange(idx_first_text_block, len(block_edges), 2)
|
|
blocks = []
|
|
slice_ranges = zip(block_edges[:-1], block_edges[1:])
|
|
for i, (start, end) in enumerate(slice_ranges):
|
|
block_label = 'text' if i in idx_text_block else 'code'
|
|
# subtract 1 from indices b/c line numbers start at 1, not 0
|
|
content = ''.join(source_lines[start-1:end-1])
|
|
blocks.append((block_label, (start, end), content))
|
|
return blocks
|
|
|
|
|
|
def get_block_edges(source_file):
|
|
"""Return starting line numbers of code and text blocks
|
|
|
|
Returns
|
|
-------
|
|
block_edges : list of int
|
|
Line number for the start of each block. Note the
|
|
idx_first_text_block : {0 | 1}
|
|
0 if first block is text then, else 1 (second block better be text).
|
|
"""
|
|
block_edges = []
|
|
with open(source_file) as f:
|
|
token_iter = tokenize.generate_tokens(f.readline)
|
|
for token_tuple in token_iter:
|
|
t_id, t_str, (srow, scol), (erow, ecol), src_line = token_tuple
|
|
if (token.tok_name[t_id] == 'STRING' and scol == 0):
|
|
# Add one point to line after text (for later slicing)
|
|
block_edges.extend((srow, erow+1))
|
|
idx_first_text_block = 0
|
|
# when example doesn't start with text block.
|
|
if not block_edges[0] == 1:
|
|
block_edges.insert(0, 1)
|
|
idx_first_text_block = 1
|
|
# when example doesn't end with text block.
|
|
if not block_edges[-1] == erow: # iffy: I'm using end state of loop
|
|
block_edges.append(erow)
|
|
return block_edges, idx_first_text_block
|
|
|
|
|
|
def process_blocks(blocks, src_path, image_path, cfg):
|
|
"""Run source, save plots as images, and convert blocks to rst.
|
|
|
|
Parameters
|
|
----------
|
|
blocks : list of block tuples
|
|
Code and text blocks from example. See `split_code_and_text_blocks`.
|
|
src_path : str
|
|
Path to example file.
|
|
image_path : str
|
|
Path where plots are saved (format string which accepts figure number).
|
|
cfg : config object
|
|
Sphinx config object created by Sphinx.
|
|
|
|
Returns
|
|
-------
|
|
figure_list : list
|
|
List of figure names saved by the example.
|
|
rst_text : str
|
|
Text with code wrapped code-block directives.
|
|
"""
|
|
src_dir, src_name = src_path.psplit()
|
|
if not src_name.startswith('plot'):
|
|
return [], ''
|
|
|
|
# index of blocks which have inline plots
|
|
inline_tag = cfg.plot2rst_plot_tag
|
|
idx_inline_plot = [i for i, b in enumerate(blocks)
|
|
if inline_tag in b[2]]
|
|
|
|
image_dir, image_fmt_str = image_path.psplit()
|
|
|
|
figure_list = []
|
|
plt.rcdefaults()
|
|
plt.rcParams.update(cfg.plot2rst_rcparams)
|
|
plt.close('all')
|
|
|
|
example_globals = {}
|
|
rst_blocks = []
|
|
fig_num = 1
|
|
for i, (blabel, brange, bcontent) in enumerate(blocks):
|
|
if blabel == 'code':
|
|
exec(bcontent, example_globals)
|
|
rst_blocks.append(codestr2rst(bcontent))
|
|
else:
|
|
if i in idx_inline_plot:
|
|
plt.savefig(image_path.format(fig_num))
|
|
figure_name = image_fmt_str.format(fig_num)
|
|
fig_num += 1
|
|
figure_list.append(figure_name)
|
|
figure_link = os.path.join('images', figure_name)
|
|
bcontent = bcontent.replace(inline_tag, figure_link)
|
|
rst_blocks.append(docstr2rst(bcontent))
|
|
return figure_list, '\n'.join(rst_blocks)
|
|
|
|
|
|
def codestr2rst(codestr):
|
|
"""Return reStructuredText code block from code string"""
|
|
code_directive = ".. code-block:: python\n\n"
|
|
indented_block = '\t' + codestr.replace('\n', '\n\t')
|
|
return code_directive + indented_block
|
|
|
|
|
|
def docstr2rst(docstr):
|
|
"""Return reStructuredText from docstring"""
|
|
idx_whitespace = len(docstr.rstrip()) - len(docstr)
|
|
whitespace = docstr[idx_whitespace:]
|
|
return eval(docstr) + whitespace
|
|
|
|
|
|
def save_all_figures(image_path):
|
|
"""Save all matplotlib figures.
|
|
|
|
Parameters
|
|
----------
|
|
image_path : str
|
|
Path where plots are saved (format string which accepts figure number).
|
|
"""
|
|
figure_list = []
|
|
image_dir, image_fmt_str = image_path.psplit()
|
|
fig_mngr = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
|
|
for fig_num in (m.num for m in fig_mngr):
|
|
# Set the fig_num figure as the current figure as we can't
|
|
# save a figure that's not the current figure.
|
|
plt.figure(fig_num)
|
|
plt.savefig(image_path.format(fig_num))
|
|
figure_list.append(image_fmt_str.format(fig_num))
|
|
return figure_list
|