494 lines
18 KiB
Python
494 lines
18 KiB
Python
|
#!/usr/bin/env python3
|
||
|
"""
|
||
|
runtests.py [OPTIONS] [-- ARGS]
|
||
|
|
||
|
Run tests, building the project first.
|
||
|
|
||
|
Examples::
|
||
|
|
||
|
$ python runtests.py
|
||
|
$ python runtests.py -s {SAMPLE_SUBMODULE}
|
||
|
$ python runtests.py -t {SAMPLE_TEST}
|
||
|
$ python runtests.py --ipython
|
||
|
$ python runtests.py --python somescript.py
|
||
|
$ python runtests.py --bench
|
||
|
$ python runtests.py --durations 20
|
||
|
|
||
|
Run a debugger:
|
||
|
|
||
|
$ gdb --args python runtests.py [...other args...]
|
||
|
|
||
|
Disable pytest capturing of output by using its '-s' option:
|
||
|
|
||
|
$ python runtests.py -- -s
|
||
|
|
||
|
Generate C code coverage listing under build/lcov/:
|
||
|
(requires http://ltp.sourceforge.net/coverage/lcov.php)
|
||
|
|
||
|
$ python runtests.py --gcov [...other args...]
|
||
|
$ python runtests.py --lcov-html
|
||
|
|
||
|
"""
|
||
|
#
|
||
|
# This is a generic test runner script for projects using NumPy's test
|
||
|
# framework. Change the following values to adapt to your project:
|
||
|
#
|
||
|
|
||
|
PROJECT_MODULE = "numpy"
|
||
|
PROJECT_ROOT_FILES = ['numpy', 'LICENSE.txt', 'setup.py']
|
||
|
SAMPLE_TEST = "numpy/linalg/tests/test_linalg.py::test_byteorder_check"
|
||
|
SAMPLE_SUBMODULE = "linalg"
|
||
|
|
||
|
EXTRA_PATH = ['/usr/lib/ccache', '/usr/lib/f90cache',
|
||
|
'/usr/local/lib/ccache', '/usr/local/lib/f90cache']
|
||
|
|
||
|
# ---------------------------------------------------------------------
|
||
|
|
||
|
|
||
|
if __doc__ is None:
|
||
|
__doc__ = "Run without -OO if you want usage info"
|
||
|
else:
|
||
|
__doc__ = __doc__.format(**globals())
|
||
|
|
||
|
|
||
|
import sys
|
||
|
import os
|
||
|
|
||
|
# In case we are run from the source directory, we don't want to import the
|
||
|
# project from there:
|
||
|
sys.path.pop(0)
|
||
|
|
||
|
import shutil
|
||
|
import subprocess
|
||
|
import time
|
||
|
from argparse import ArgumentParser, REMAINDER
|
||
|
|
||
|
ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
|
||
|
|
||
|
def main(argv):
|
||
|
parser = ArgumentParser(usage=__doc__.lstrip())
|
||
|
parser.add_argument("--verbose", "-v", action="count", default=1,
|
||
|
help="more verbosity")
|
||
|
parser.add_argument("--debug-info", action="store_true",
|
||
|
help=("add --verbose-cfg to build_src to show compiler "
|
||
|
"configuration output while creating "
|
||
|
"_numpyconfig.h and config.h"))
|
||
|
parser.add_argument("--no-build", "-n", action="store_true", default=False,
|
||
|
help="do not build the project (use system installed version)")
|
||
|
parser.add_argument("--build-only", "-b", action="store_true", default=False,
|
||
|
help="just build, do not run any tests")
|
||
|
parser.add_argument("--doctests", action="store_true", default=False,
|
||
|
help="Run doctests in module")
|
||
|
parser.add_argument("--refguide-check", action="store_true", default=False,
|
||
|
help="Run refguide (doctest) check (do not run regular tests.)")
|
||
|
parser.add_argument("--coverage", action="store_true", default=False,
|
||
|
help=("report coverage of project code. HTML output goes "
|
||
|
"under build/coverage"))
|
||
|
parser.add_argument("--durations", action="store", default=-1, type=int,
|
||
|
help=("Time N slowest tests, time all if 0, time none if < 0"))
|
||
|
parser.add_argument("--gcov", action="store_true", default=False,
|
||
|
help=("enable C code coverage via gcov (requires GCC). "
|
||
|
"gcov output goes to build/**/*.gc*"))
|
||
|
parser.add_argument("--lcov-html", action="store_true", default=False,
|
||
|
help=("produce HTML for C code coverage information "
|
||
|
"from a previous run with --gcov. "
|
||
|
"HTML output goes to build/lcov/"))
|
||
|
parser.add_argument("--mode", "-m", default="fast",
|
||
|
help="'fast', 'full', or something that could be "
|
||
|
"passed to nosetests -A [default: fast]")
|
||
|
parser.add_argument("--submodule", "-s", default=None,
|
||
|
help="Submodule whose tests to run (cluster, constants, ...)")
|
||
|
parser.add_argument("--pythonpath", "-p", default=None,
|
||
|
help="Paths to prepend to PYTHONPATH")
|
||
|
parser.add_argument("--tests", "-t", action='append',
|
||
|
help="Specify tests to run")
|
||
|
parser.add_argument("--python", action="store_true",
|
||
|
help="Start a Python shell with PYTHONPATH set")
|
||
|
parser.add_argument("--ipython", "-i", action="store_true",
|
||
|
help="Start IPython shell with PYTHONPATH set")
|
||
|
parser.add_argument("--shell", action="store_true",
|
||
|
help="Start Unix shell with PYTHONPATH set")
|
||
|
parser.add_argument("--debug", "-g", action="store_true",
|
||
|
help="Debug build")
|
||
|
parser.add_argument("--parallel", "-j", type=int, default=0,
|
||
|
help="Number of parallel jobs during build")
|
||
|
parser.add_argument("--warn-error", action="store_true",
|
||
|
help="Set -Werror to convert all compiler warnings to errors")
|
||
|
parser.add_argument("--show-build-log", action="store_true",
|
||
|
help="Show build output rather than using a log file")
|
||
|
parser.add_argument("--bench", action="store_true",
|
||
|
help="Run benchmark suite instead of test suite")
|
||
|
parser.add_argument("--bench-compare", action="store", metavar="COMMIT",
|
||
|
help=("Compare benchmark results of current HEAD to "
|
||
|
"BEFORE. Use an additional "
|
||
|
"--bench-compare=COMMIT to override HEAD with "
|
||
|
"COMMIT. Note that you need to commit your "
|
||
|
"changes first!"))
|
||
|
parser.add_argument("args", metavar="ARGS", default=[], nargs=REMAINDER,
|
||
|
help="Arguments to pass to Nose, asv, Python or shell")
|
||
|
args = parser.parse_args(argv)
|
||
|
|
||
|
if args.durations < 0:
|
||
|
args.durations = -1
|
||
|
|
||
|
if args.bench_compare:
|
||
|
args.bench = True
|
||
|
args.no_build = True # ASV does the building
|
||
|
|
||
|
if args.lcov_html:
|
||
|
# generate C code coverage output
|
||
|
lcov_generate()
|
||
|
sys.exit(0)
|
||
|
|
||
|
if args.pythonpath:
|
||
|
for p in reversed(args.pythonpath.split(os.pathsep)):
|
||
|
sys.path.insert(0, p)
|
||
|
|
||
|
if args.gcov:
|
||
|
gcov_reset_counters()
|
||
|
|
||
|
if args.debug and args.bench:
|
||
|
print("*** Benchmarks should not be run against debug "
|
||
|
"version; remove -g flag ***")
|
||
|
|
||
|
if not args.no_build:
|
||
|
# we need the noarch path in case the package is pure python.
|
||
|
site_dir, site_dir_noarch = build_project(args)
|
||
|
sys.path.insert(0, site_dir)
|
||
|
sys.path.insert(0, site_dir_noarch)
|
||
|
os.environ['PYTHONPATH'] = site_dir + os.pathsep + site_dir_noarch
|
||
|
else:
|
||
|
_temp = __import__(PROJECT_MODULE)
|
||
|
site_dir = os.path.sep.join(_temp.__file__.split(os.path.sep)[:-2])
|
||
|
|
||
|
extra_argv = args.args[:]
|
||
|
if not args.bench:
|
||
|
# extra_argv may also lists selected benchmarks
|
||
|
if extra_argv and extra_argv[0] == '--':
|
||
|
extra_argv = extra_argv[1:]
|
||
|
|
||
|
if args.python:
|
||
|
# Debugging issues with warnings is much easier if you can see them
|
||
|
print("Enabling display of all warnings")
|
||
|
import warnings
|
||
|
import types
|
||
|
|
||
|
warnings.filterwarnings("always")
|
||
|
if extra_argv:
|
||
|
# Don't use subprocess, since we don't want to include the
|
||
|
# current path in PYTHONPATH.
|
||
|
sys.argv = extra_argv
|
||
|
with open(extra_argv[0], 'r') as f:
|
||
|
script = f.read()
|
||
|
sys.modules['__main__'] = types.ModuleType('__main__')
|
||
|
ns = dict(__name__='__main__',
|
||
|
__file__=extra_argv[0])
|
||
|
exec(script, ns)
|
||
|
sys.exit(0)
|
||
|
else:
|
||
|
import code
|
||
|
code.interact()
|
||
|
sys.exit(0)
|
||
|
|
||
|
if args.ipython:
|
||
|
# Debugging issues with warnings is much easier if you can see them
|
||
|
print("Enabling display of all warnings and pre-importing numpy as np")
|
||
|
import warnings; warnings.filterwarnings("always")
|
||
|
import IPython
|
||
|
import numpy as np
|
||
|
IPython.embed(user_ns={"np": np})
|
||
|
sys.exit(0)
|
||
|
|
||
|
if args.shell:
|
||
|
shell = os.environ.get('SHELL', 'cmd' if os.name == 'nt' else 'sh')
|
||
|
print("Spawning a shell ({})...".format(shell))
|
||
|
subprocess.call([shell] + extra_argv)
|
||
|
sys.exit(0)
|
||
|
|
||
|
if args.coverage:
|
||
|
dst_dir = os.path.join(ROOT_DIR, 'build', 'coverage')
|
||
|
fn = os.path.join(dst_dir, 'coverage_html.js')
|
||
|
if os.path.isdir(dst_dir) and os.path.isfile(fn):
|
||
|
shutil.rmtree(dst_dir)
|
||
|
extra_argv += ['--cov-report=html:' + dst_dir]
|
||
|
|
||
|
if args.refguide_check:
|
||
|
cmd = [os.path.join(ROOT_DIR, 'tools', 'refguide_check.py'),
|
||
|
'--doctests']
|
||
|
if args.submodule:
|
||
|
cmd += [args.submodule]
|
||
|
os.execv(sys.executable, [sys.executable] + cmd)
|
||
|
sys.exit(0)
|
||
|
|
||
|
if args.bench:
|
||
|
# Run ASV
|
||
|
for i, v in enumerate(extra_argv):
|
||
|
if v.startswith("--"):
|
||
|
items = extra_argv[:i]
|
||
|
if v == "--":
|
||
|
i += 1 # skip '--' indicating further are passed on.
|
||
|
bench_args = extra_argv[i:]
|
||
|
break
|
||
|
else:
|
||
|
items = extra_argv
|
||
|
bench_args = []
|
||
|
|
||
|
if args.tests:
|
||
|
items += args.tests
|
||
|
if args.submodule:
|
||
|
items += [args.submodule]
|
||
|
for a in items:
|
||
|
bench_args.extend(['--bench', a])
|
||
|
|
||
|
if not args.bench_compare:
|
||
|
cmd = ['asv', 'run', '-n', '-e', '--python=same'] + bench_args
|
||
|
ret = subprocess.call(cmd, cwd=os.path.join(ROOT_DIR, 'benchmarks'))
|
||
|
sys.exit(ret)
|
||
|
else:
|
||
|
commits = [x.strip() for x in args.bench_compare.split(',')]
|
||
|
if len(commits) == 1:
|
||
|
commit_a = commits[0]
|
||
|
commit_b = 'HEAD'
|
||
|
elif len(commits) == 2:
|
||
|
commit_a, commit_b = commits
|
||
|
else:
|
||
|
p.error("Too many commits to compare benchmarks for")
|
||
|
|
||
|
# Check for uncommitted files
|
||
|
if commit_b == 'HEAD':
|
||
|
r1 = subprocess.call(['git', 'diff-index', '--quiet',
|
||
|
'--cached', 'HEAD'])
|
||
|
r2 = subprocess.call(['git', 'diff-files', '--quiet'])
|
||
|
if r1 != 0 or r2 != 0:
|
||
|
print("*"*80)
|
||
|
print("WARNING: you have uncommitted changes --- "
|
||
|
"these will NOT be benchmarked!")
|
||
|
print("*"*80)
|
||
|
|
||
|
# Fix commit ids (HEAD is local to current repo)
|
||
|
out = subprocess.check_output(['git', 'rev-parse', commit_b])
|
||
|
commit_b = out.strip().decode('ascii')
|
||
|
|
||
|
out = subprocess.check_output(['git', 'rev-parse', commit_a])
|
||
|
commit_a = out.strip().decode('ascii')
|
||
|
|
||
|
cmd = ['asv', 'continuous', '-e', '-f', '1.05',
|
||
|
commit_a, commit_b] + bench_args
|
||
|
ret = subprocess.call(cmd, cwd=os.path.join(ROOT_DIR, 'benchmarks'))
|
||
|
sys.exit(ret)
|
||
|
|
||
|
if args.build_only:
|
||
|
sys.exit(0)
|
||
|
else:
|
||
|
__import__(PROJECT_MODULE)
|
||
|
test = sys.modules[PROJECT_MODULE].test
|
||
|
|
||
|
if args.submodule:
|
||
|
tests = [PROJECT_MODULE + "." + args.submodule]
|
||
|
elif args.tests:
|
||
|
tests = args.tests
|
||
|
else:
|
||
|
tests = None
|
||
|
|
||
|
|
||
|
# Run the tests under build/test
|
||
|
|
||
|
if not args.no_build:
|
||
|
test_dir = site_dir
|
||
|
else:
|
||
|
test_dir = os.path.join(ROOT_DIR, 'build', 'test')
|
||
|
if not os.path.isdir(test_dir):
|
||
|
os.makedirs(test_dir)
|
||
|
|
||
|
shutil.copyfile(os.path.join(ROOT_DIR, '.coveragerc'),
|
||
|
os.path.join(test_dir, '.coveragerc'))
|
||
|
|
||
|
cwd = os.getcwd()
|
||
|
try:
|
||
|
os.chdir(test_dir)
|
||
|
result = test(args.mode,
|
||
|
verbose=args.verbose,
|
||
|
extra_argv=extra_argv,
|
||
|
doctests=args.doctests,
|
||
|
coverage=args.coverage,
|
||
|
durations=args.durations,
|
||
|
tests=tests)
|
||
|
finally:
|
||
|
os.chdir(cwd)
|
||
|
|
||
|
if isinstance(result, bool):
|
||
|
sys.exit(0 if result else 1)
|
||
|
elif result.wasSuccessful():
|
||
|
sys.exit(0)
|
||
|
else:
|
||
|
sys.exit(1)
|
||
|
|
||
|
|
||
|
def build_project(args):
|
||
|
"""
|
||
|
Build a dev version of the project.
|
||
|
|
||
|
Returns
|
||
|
-------
|
||
|
site_dir
|
||
|
site-packages directory where it was installed
|
||
|
|
||
|
"""
|
||
|
|
||
|
import distutils.sysconfig
|
||
|
|
||
|
root_ok = [os.path.exists(os.path.join(ROOT_DIR, fn))
|
||
|
for fn in PROJECT_ROOT_FILES]
|
||
|
if not all(root_ok):
|
||
|
print("To build the project, run runtests.py in "
|
||
|
"git checkout or unpacked source")
|
||
|
sys.exit(1)
|
||
|
|
||
|
dst_dir = os.path.join(ROOT_DIR, 'build', 'testenv')
|
||
|
|
||
|
env = dict(os.environ)
|
||
|
cmd = [sys.executable, 'setup.py']
|
||
|
|
||
|
# Always use ccache, if installed
|
||
|
env['PATH'] = os.pathsep.join(EXTRA_PATH + env.get('PATH', '').split(os.pathsep))
|
||
|
cvars = distutils.sysconfig.get_config_vars()
|
||
|
compiler = env.get('CC') or cvars.get('CC', '')
|
||
|
if 'gcc' in compiler:
|
||
|
# Check that this isn't clang masquerading as gcc.
|
||
|
if sys.platform != 'darwin' or 'gnu-gcc' in compiler:
|
||
|
# add flags used as werrors
|
||
|
warnings_as_errors = ' '.join([
|
||
|
# from tools/travis-test.sh
|
||
|
'-Werror=vla',
|
||
|
'-Werror=nonnull',
|
||
|
'-Werror=pointer-arith',
|
||
|
'-Wlogical-op',
|
||
|
# from sysconfig
|
||
|
'-Werror=unused-function',
|
||
|
])
|
||
|
env['CFLAGS'] = warnings_as_errors + ' ' + env.get('CFLAGS', '')
|
||
|
if args.debug or args.gcov:
|
||
|
# assume everyone uses gcc/gfortran
|
||
|
env['OPT'] = '-O0 -ggdb'
|
||
|
env['FOPT'] = '-O0 -ggdb'
|
||
|
if args.gcov:
|
||
|
env['OPT'] = '-O0 -ggdb'
|
||
|
env['FOPT'] = '-O0 -ggdb'
|
||
|
env['CC'] = cvars['CC'] + ' --coverage'
|
||
|
env['CXX'] = cvars['CXX'] + ' --coverage'
|
||
|
env['F77'] = 'gfortran --coverage '
|
||
|
env['F90'] = 'gfortran --coverage '
|
||
|
env['LDSHARED'] = cvars['LDSHARED'] + ' --coverage'
|
||
|
env['LDFLAGS'] = " ".join(cvars['LDSHARED'].split()[1:]) + ' --coverage'
|
||
|
|
||
|
cmd += ["build"]
|
||
|
if args.parallel > 1:
|
||
|
cmd += ["-j", str(args.parallel)]
|
||
|
if args.debug_info:
|
||
|
cmd += ["build_src", "--verbose-cfg"]
|
||
|
if args.warn_error:
|
||
|
cmd += ["--warn-error"]
|
||
|
# Install; avoid producing eggs so numpy can be imported from dst_dir.
|
||
|
cmd += ['install', '--prefix=' + dst_dir,
|
||
|
'--single-version-externally-managed',
|
||
|
'--record=' + dst_dir + 'tmp_install_log.txt']
|
||
|
|
||
|
from distutils.sysconfig import get_python_lib
|
||
|
site_dir = get_python_lib(prefix=dst_dir, plat_specific=True)
|
||
|
site_dir_noarch = get_python_lib(prefix=dst_dir, plat_specific=False)
|
||
|
# easy_install won't install to a path that Python by default cannot see
|
||
|
# and isn't on the PYTHONPATH. Plus, it has to exist.
|
||
|
if not os.path.exists(site_dir):
|
||
|
os.makedirs(site_dir)
|
||
|
if not os.path.exists(site_dir_noarch):
|
||
|
os.makedirs(site_dir_noarch)
|
||
|
env['PYTHONPATH'] = site_dir + ':' + site_dir_noarch
|
||
|
|
||
|
log_filename = os.path.join(ROOT_DIR, 'build.log')
|
||
|
|
||
|
if args.show_build_log:
|
||
|
ret = subprocess.call(cmd, env=env, cwd=ROOT_DIR)
|
||
|
else:
|
||
|
log_filename = os.path.join(ROOT_DIR, 'build.log')
|
||
|
print("Building, see build.log...")
|
||
|
with open(log_filename, 'w') as log:
|
||
|
p = subprocess.Popen(cmd, env=env, stdout=log, stderr=log,
|
||
|
cwd=ROOT_DIR)
|
||
|
try:
|
||
|
# Wait for it to finish, and print something to indicate the
|
||
|
# process is alive, but only if the log file has grown (to
|
||
|
# allow continuous integration environments kill a hanging
|
||
|
# process accurately if it produces no output)
|
||
|
last_blip = time.time()
|
||
|
last_log_size = os.stat(log_filename).st_size
|
||
|
while p.poll() is None:
|
||
|
time.sleep(0.5)
|
||
|
if time.time() - last_blip > 60:
|
||
|
log_size = os.stat(log_filename).st_size
|
||
|
if log_size > last_log_size:
|
||
|
print(" ... build in progress")
|
||
|
last_blip = time.time()
|
||
|
last_log_size = log_size
|
||
|
|
||
|
ret = p.wait()
|
||
|
except:
|
||
|
p.kill()
|
||
|
p.wait()
|
||
|
raise
|
||
|
|
||
|
if ret == 0:
|
||
|
print("Build OK")
|
||
|
else:
|
||
|
if not args.show_build_log:
|
||
|
with open(log_filename, 'r') as f:
|
||
|
print(f.read())
|
||
|
print("Build failed!")
|
||
|
sys.exit(1)
|
||
|
|
||
|
return site_dir, site_dir_noarch
|
||
|
|
||
|
|
||
|
#
|
||
|
# GCOV support
|
||
|
#
|
||
|
def gcov_reset_counters():
|
||
|
print("Removing previous GCOV .gcda files...")
|
||
|
build_dir = os.path.join(ROOT_DIR, 'build')
|
||
|
for dirpath, dirnames, filenames in os.walk(build_dir):
|
||
|
for fn in filenames:
|
||
|
if fn.endswith('.gcda') or fn.endswith('.da'):
|
||
|
pth = os.path.join(dirpath, fn)
|
||
|
os.unlink(pth)
|
||
|
|
||
|
#
|
||
|
# LCOV support
|
||
|
#
|
||
|
|
||
|
LCOV_OUTPUT_FILE = os.path.join(ROOT_DIR, 'build', 'lcov.out')
|
||
|
LCOV_HTML_DIR = os.path.join(ROOT_DIR, 'build', 'lcov')
|
||
|
|
||
|
def lcov_generate():
|
||
|
try: os.unlink(LCOV_OUTPUT_FILE)
|
||
|
except OSError: pass
|
||
|
try: shutil.rmtree(LCOV_HTML_DIR)
|
||
|
except OSError: pass
|
||
|
|
||
|
print("Capturing lcov info...")
|
||
|
subprocess.call(['lcov', '-q', '-c',
|
||
|
'-d', os.path.join(ROOT_DIR, 'build'),
|
||
|
'-b', ROOT_DIR,
|
||
|
'--output-file', LCOV_OUTPUT_FILE])
|
||
|
|
||
|
print("Generating lcov HTML output...")
|
||
|
ret = subprocess.call(['genhtml', '-q', LCOV_OUTPUT_FILE,
|
||
|
'--output-directory', LCOV_HTML_DIR,
|
||
|
'--legend', '--highlight'])
|
||
|
if ret != 0:
|
||
|
print("genhtml failed!")
|
||
|
else:
|
||
|
print("HTML output generated under build/lcov/")
|
||
|
|
||
|
|
||
|
if __name__ == "__main__":
|
||
|
main(argv=sys.argv[1:])
|