328 lines
11 KiB
Python
328 lines
11 KiB
Python
import glob
|
|
import hashlib
|
|
import os
|
|
import platform
|
|
import sys
|
|
import shutil
|
|
import tarfile
|
|
import textwrap
|
|
import zipfile
|
|
|
|
from tempfile import mkstemp, gettempdir
|
|
from urllib.request import urlopen, Request
|
|
from urllib.error import HTTPError
|
|
|
|
OPENBLAS_V = '0.3.13'
|
|
OPENBLAS_LONG = 'v0.3.13'
|
|
BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs'
|
|
BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download'
|
|
ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64',
|
|
'i686', 'ppc64le', 's390x']
|
|
IS_32BIT = sys.maxsize < 2**32
|
|
|
|
|
|
def get_arch():
|
|
if platform.system() == 'Windows':
|
|
ret = 'windows'
|
|
elif platform.system() == 'Darwin':
|
|
ret = 'darwin'
|
|
else:
|
|
ret = platform.uname().machine
|
|
# What do 32 bit machines report?
|
|
# If they are a docker, they can report x86_64
|
|
if 'x86' in ret and IS_32BIT:
|
|
ret = 'i686'
|
|
assert ret in ARCHITECTURES, f'invalid architecture {ret}'
|
|
return ret
|
|
|
|
|
|
def get_ilp64():
|
|
if os.environ.get("NPY_USE_BLAS_ILP64", "0") == "0":
|
|
return None
|
|
if IS_32BIT:
|
|
raise RuntimeError("NPY_USE_BLAS_ILP64 set on 32-bit arch")
|
|
return "64_"
|
|
|
|
|
|
def get_manylinux(arch):
|
|
if arch in ('x86_64', 'i686'):
|
|
default = '2010'
|
|
else:
|
|
default = '2014'
|
|
ret = os.environ.get("MB_ML_VER", default)
|
|
# XXX For PEP 600 this can be a glibc version
|
|
assert ret in ('1', '2010', '2014'), f'invalid MB_ML_VER {ret}'
|
|
return ret
|
|
|
|
|
|
def download_openblas(target, arch, ilp64, is_32bit):
|
|
ml_ver = get_manylinux(arch)
|
|
fnsuffix = {None: "", "64_": "64_"}[ilp64]
|
|
filename = ''
|
|
headers = {'User-Agent':
|
|
('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; '
|
|
'(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3')}
|
|
if arch in ('aarch64', 'ppc64le', 's390x', 'x86_64', 'i686'):
|
|
suffix = f'manylinux{ml_ver}_{arch}.tar.gz'
|
|
filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
|
|
typ = 'tar.gz'
|
|
elif arch == 'darwin':
|
|
suffix = 'macosx_10_9_x86_64-gf_1becaaa.tar.gz'
|
|
filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
|
|
typ = 'tar.gz'
|
|
elif arch == 'windows':
|
|
if is_32bit:
|
|
suffix = 'win32-gcc_8_1_0.zip'
|
|
else:
|
|
suffix = 'win_amd64-gcc_8_1_0.zip'
|
|
filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}'
|
|
typ = 'zip'
|
|
if not filename:
|
|
return None
|
|
req = Request(url=filename, headers=headers)
|
|
try:
|
|
response = urlopen(req)
|
|
except HTTPError:
|
|
print(f'Could not download "{filename}"', file=sys.stderr)
|
|
raise
|
|
length = response.getheader('content-length')
|
|
if response.status != 200:
|
|
print(f'Could not download "{filename}"', file=sys.stderr)
|
|
return None
|
|
print(f"Downloading {length} from {filename}", file=sys.stderr)
|
|
data = response.read()
|
|
# Verify hash
|
|
key = os.path.basename(filename)
|
|
print("Saving to file", file=sys.stderr)
|
|
with open(target, 'wb') as fid:
|
|
fid.write(data)
|
|
return typ
|
|
|
|
|
|
def setup_openblas(arch=get_arch(), ilp64=get_ilp64(), is_32bit=IS_32BIT):
|
|
'''
|
|
Download and setup an openblas library for building. If successful,
|
|
the configuration script will find it automatically.
|
|
|
|
Returns
|
|
-------
|
|
msg : str
|
|
path to extracted files on success, otherwise indicates what went wrong
|
|
To determine success, do ``os.path.exists(msg)``
|
|
'''
|
|
_, tmp = mkstemp()
|
|
if not arch:
|
|
raise ValueError('unknown architecture')
|
|
typ = download_openblas(tmp, arch, ilp64, is_32bit)
|
|
if not typ:
|
|
return ''
|
|
if arch == 'windows':
|
|
if not typ == 'zip':
|
|
return f'expecting to download zipfile on windows, not {typ}'
|
|
return unpack_windows_zip(tmp)
|
|
else:
|
|
if not typ == 'tar.gz':
|
|
return 'expecting to download tar.gz, not %s' % str(typ)
|
|
return unpack_targz(tmp)
|
|
|
|
|
|
def unpack_windows_zip(fname):
|
|
with zipfile.ZipFile(fname, 'r') as zf:
|
|
# Get the openblas.a file, but not openblas.dll.a nor openblas.dev.a
|
|
lib = [x for x in zf.namelist() if OPENBLAS_LONG in x and
|
|
x.endswith('a') and not x.endswith('dll.a') and
|
|
not x.endswith('dev.a')]
|
|
if not lib:
|
|
return 'could not find libopenblas_%s*.a ' \
|
|
'in downloaded zipfile' % OPENBLAS_LONG
|
|
target = os.path.join(gettempdir(), 'openblas.a')
|
|
with open(target, 'wb') as fid:
|
|
fid.write(zf.read(lib[0]))
|
|
return target
|
|
|
|
|
|
def unpack_targz(fname):
|
|
target = os.path.join(gettempdir(), 'openblas')
|
|
if not os.path.exists(target):
|
|
os.mkdir(target)
|
|
with tarfile.open(fname, 'r') as zf:
|
|
# Strip common prefix from paths when unpacking
|
|
prefix = os.path.commonpath(zf.getnames())
|
|
extract_tarfile_to(zf, target, prefix)
|
|
return target
|
|
|
|
|
|
def extract_tarfile_to(tarfileobj, target_path, archive_path):
|
|
"""Extract TarFile contents under archive_path/ to target_path/"""
|
|
|
|
target_path = os.path.abspath(target_path)
|
|
|
|
def get_members():
|
|
for member in tarfileobj.getmembers():
|
|
if archive_path:
|
|
norm_path = os.path.normpath(member.name)
|
|
if norm_path.startswith(archive_path + os.path.sep):
|
|
member.name = norm_path[len(archive_path)+1:]
|
|
else:
|
|
continue
|
|
|
|
dst_path = os.path.abspath(os.path.join(target_path, member.name))
|
|
if os.path.commonpath([target_path, dst_path]) != target_path:
|
|
# Path not under target_path, probably contains ../
|
|
continue
|
|
|
|
yield member
|
|
|
|
tarfileobj.extractall(target_path, members=get_members())
|
|
|
|
|
|
def make_init(dirname):
|
|
'''
|
|
Create a _distributor_init.py file for OpenBlas
|
|
'''
|
|
with open(os.path.join(dirname, '_distributor_init.py'), 'wt') as fid:
|
|
fid.write(textwrap.dedent("""
|
|
'''
|
|
Helper to preload windows dlls to prevent dll not found errors.
|
|
Once a DLL is preloaded, its namespace is made available to any
|
|
subsequent DLL. This file originated in the numpy-wheels repo,
|
|
and is created as part of the scripts that build the wheel.
|
|
'''
|
|
import os
|
|
import glob
|
|
if os.name == 'nt':
|
|
# convention for storing / loading the DLL from
|
|
# numpy/.libs/, if present
|
|
try:
|
|
from ctypes import WinDLL
|
|
basedir = os.path.dirname(__file__)
|
|
except:
|
|
pass
|
|
else:
|
|
libs_dir = os.path.abspath(os.path.join(basedir, '.libs'))
|
|
DLL_filenames = []
|
|
if os.path.isdir(libs_dir):
|
|
for filename in glob.glob(os.path.join(libs_dir,
|
|
'*openblas*dll')):
|
|
# NOTE: would it change behavior to load ALL
|
|
# DLLs at this path vs. the name restriction?
|
|
WinDLL(os.path.abspath(filename))
|
|
DLL_filenames.append(filename)
|
|
if len(DLL_filenames) > 1:
|
|
import warnings
|
|
warnings.warn("loaded more than 1 DLL from .libs:"
|
|
"\\n%s" % "\\n".join(DLL_filenames),
|
|
stacklevel=1)
|
|
"""))
|
|
|
|
|
|
def test_setup(arches):
|
|
'''
|
|
Make sure all the downloadable files exist and can be opened
|
|
'''
|
|
def items():
|
|
""" yields all combinations of arch, ilp64, is_32bit
|
|
"""
|
|
for arch in arches:
|
|
yield arch, None, False
|
|
if arch not in ('i686',):
|
|
yield arch, '64_', False
|
|
if arch in ('windows',):
|
|
yield arch, None, True
|
|
if arch in ('i686', 'x86_64'):
|
|
oldval = os.environ.get('MB_ML_VER', None)
|
|
os.environ['MB_ML_VER'] = '1'
|
|
yield arch, None, False
|
|
# Once we create x86_64 and i686 manylinux2014 wheels...
|
|
# os.environ['MB_ML_VER'] = '2014'
|
|
# yield arch, None, False
|
|
if oldval:
|
|
os.environ['MB_ML_VER'] = oldval
|
|
else:
|
|
os.environ.pop('MB_ML_VER')
|
|
|
|
errs = []
|
|
for arch, ilp64, is_32bit in items():
|
|
if arch == '':
|
|
continue
|
|
if arch not in arches:
|
|
continue
|
|
target = None
|
|
try:
|
|
try:
|
|
target = setup_openblas(arch, ilp64, is_32bit)
|
|
except Exception as e:
|
|
print(f'Could not setup {arch} with ilp64 {ilp64}, '
|
|
f'32bit {is_32bit}:')
|
|
print(e)
|
|
errs.append(e)
|
|
continue
|
|
if not target:
|
|
raise RuntimeError(f'Could not setup {arch}')
|
|
print(target)
|
|
if arch == 'windows':
|
|
if not target.endswith('.a'):
|
|
raise RuntimeError("Not .a extracted!")
|
|
else:
|
|
files = glob.glob(os.path.join(target, "lib", "*.a"))
|
|
if not files:
|
|
raise RuntimeError("No lib/*.a unpacked!")
|
|
finally:
|
|
if target is not None:
|
|
if os.path.isfile(target):
|
|
os.unlink(target)
|
|
else:
|
|
shutil.rmtree(target)
|
|
if errs:
|
|
raise errs[0]
|
|
|
|
|
|
def test_version(expected_version, ilp64=get_ilp64()):
|
|
"""
|
|
Assert that expected OpenBLAS version is
|
|
actually available via NumPy
|
|
"""
|
|
import numpy
|
|
import ctypes
|
|
|
|
dll = ctypes.CDLL(numpy.core._multiarray_umath.__file__)
|
|
if ilp64 == "64_":
|
|
get_config = dll.openblas_get_config64_
|
|
else:
|
|
get_config = dll.openblas_get_config
|
|
get_config.restype = ctypes.c_char_p
|
|
res = get_config()
|
|
print('OpenBLAS get_config returned', str(res))
|
|
if not expected_version:
|
|
expected_version = OPENBLAS_V
|
|
check_str = b'OpenBLAS %s' % expected_version.encode()
|
|
print(check_str)
|
|
assert check_str in res, f'{expected_version} not found in {res}'
|
|
if ilp64:
|
|
assert b"USE64BITINT" in res
|
|
else:
|
|
assert b"USE64BITINT" not in res
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import argparse
|
|
parser = argparse.ArgumentParser(
|
|
description='Download and expand an OpenBLAS archive for this '
|
|
'architecture')
|
|
parser.add_argument('--test', nargs='*', default=None,
|
|
help='Test different architectures. "all", or any of '
|
|
f'{ARCHITECTURES}')
|
|
parser.add_argument('--check_version', nargs='?', default='',
|
|
help='Check provided OpenBLAS version string '
|
|
'against available OpenBLAS')
|
|
args = parser.parse_args()
|
|
if args.check_version != '':
|
|
test_version(args.check_version)
|
|
elif args.test is None:
|
|
print(setup_openblas())
|
|
else:
|
|
if len(args.test) == 0 or 'all' in args.test:
|
|
test_setup(ARCHITECTURES)
|
|
else:
|
|
test_setup(args.test)
|