[WiP] Build wheels

.github/workflows/wheels.yml

@@ -0,0 +1,46 @@
+name: Wheels
+
+#on:
+#  push:
+#    tags: ['v*.*.*']
+
+# Temporary:
+on: [push, pull_request]
+
+jobs:
+  wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      - name: Install pkg-config
+        if: runner.os == 'Windows'
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: ucrt64
+          install: mingw-w64-ucrt-x86_64-pkg-config
+
+      - name: Put MSYS2 on PATH
+        if: runner.os == 'Windows'
+        run: Add-Content $env:GITHUB_PATH "$env:RUNNER_TEMP\msys64\ucrt64\bin"
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels
+          path: wheelhouse/*.whl
+          if-no-files-found: error

cibw_before_build.sh

@@ -0,0 +1,26 @@
+set -xe
+
+# Download and install vips
+basedir=$(python download-vips.py)
+
+if [[ $RUNNER_OS == "Linux" ]]; then
+  linkname="-l:libvips.so.42"
+elif [[ $RUNNER_OS == "Windows" ]]; then
+  linkname="-lvips"
+elif [[ $RUNNER_OS == "macOS" ]]; then
+  linkname="libvips.42.dylib"
+fi
+
+mkdir -p $basedir/lib/pkgconfig
+cat > $basedir/lib/pkgconfig/vips.pc << EOL
+prefix=${basedir}
+libdir=\${prefix}/lib
+includedir=\${prefix}/include
+
+Name: vips
+Description: Image processing library
+Version: 8.15.0
+Requires:
+Libs: -L\${libdir} ${linkname}
+Cflags: -I\${includedir} -I\${includedir}/glib-2.0 -I\${libdir}/glib-2.0/include
+EOL

download-vips.py

@@ -0,0 +1,245 @@
+import glob
+import os
+import platform
+import sysconfig
+import sys
+import shutil
+import tarfile
+
+from tempfile import mkstemp
+from urllib.request import urlopen, Request
+from urllib.error import HTTPError
+
+VIPS_VERSION = '8.15.0'
+BASE_LOC = (
+    'https://github.com/kleisauke/libvips-packaging/releases'
+)
+SUPPORTED_PLATFORMS = [
+    'linux-aarch64',
+    'linux-x86_64',
+    'musllinux-aarch64',
+    'musllinux-x86_64',
+    'win-32',
+    'win-amd64',
+    'win-arm64',
+    'macosx-arm64',
+    'macosx-x86_64',
+]
+ARCH_REMAP = {
+    '32': 'x86',
+    'aarch64': 'arm64',
+    'amd64': 'x64',
+    'arm64': 'arm64',
+    'x86_64': 'x64',
+}
+
+
+def get_plat():
+    plat = sysconfig.get_platform()
+    plat_split = plat.split("-")
+    arch = plat_split[-1]
+    if arch == "win32":
+        plat = "win-32"
+    elif arch in ["universal2", "intel"]:
+        plat = f"macosx-{platform.uname().machine}"
+    elif len(plat_split) > 2:
+        plat = f"{plat_split[0]}-{arch}"
+    assert plat in SUPPORTED_PLATFORMS, f'invalid platform {plat}'
+    return plat
+
+
+def get_manylinux(arch):
+    return f'linux-{ARCH_REMAP[arch]}.tar.gz'
+
+
+def get_musllinux(arch):
+    return f'linux-musl-{ARCH_REMAP[arch]}.tar.gz'
+
+
+def get_linux(arch):
+    # best way of figuring out whether manylinux or musllinux is to look
+    # at the packaging tags. If packaging isn't installed (it's not by default)
+    # fallback to sysconfig (which may be flakier)
+    try:
+        from packaging.tags import sys_tags
+        tags = list(sys_tags())
+        plat = tags[0].platform
+    except ImportError:
+        # fallback to sysconfig for figuring out if you're using musl
+        plat = 'manylinux'
+        # value could be None
+        v = sysconfig.get_config_var('HOST_GNU_TYPE') or ''
+        if 'musl' in v:
+            plat = 'musllinux'
+
+    if 'manylinux' in plat:
+        return get_manylinux(arch)
+    elif 'musllinux' in plat:
+        return get_musllinux(arch)
+
+
+def get_macosx(arch):
+    return f'osx-{ARCH_REMAP[arch]}.tar.gz'
+
+
+def get_win32(arch):
+    return f'win-{ARCH_REMAP[arch]}.tar.gz'
+
+
+def download_vips(target, plat):
+    osname, arch = plat.split("-")
+    headers = {'User-Agent':
+               ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; '
+                '(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3')}
+    suffix = None
+    if osname == "linux":
+        suffix = get_linux(arch)
+        typ = 'tar.gz'
+    elif osname == "musllinux":
+        suffix = get_musllinux(arch)
+        typ = 'tar.gz'
+    elif osname == 'macosx':
+        suffix = get_macosx(arch)
+        typ = 'tar.gz'
+    elif osname == 'win':
+        suffix = get_win32(arch)
+        typ = 'tar.gz'
+
+    if not suffix:
+        return None
+
+    filename = (
+        f'{BASE_LOC}/download/v{VIPS_VERSION}/'
+        f'libvips-{VIPS_VERSION}-{suffix}'
+    )
+    print(f'Attempting to download {filename}', file=sys.stderr)
+    req = Request(url=filename, headers=headers)
+    try:
+        response = urlopen(req)
+    except HTTPError:
+        print(f'Could not download "{filename}"', file=sys.stderr)
+        raise
+    # length = response.getheader('content-length')
+    if response.status != 200:
+        print(f'Could not download "{filename}"', file=sys.stderr)
+        return None
+    # print(f"Downloading {length} from {filename}", file=sys.stderr)
+    data = response.read()
+    # print("Saving to file", file=sys.stderr)
+    with open(target, 'wb') as fid:
+        fid.write(data)
+    return typ
+
+
+def setup_vips(plat=get_plat()):
+    '''
+    Download and setup a libvips library for building. If successful,
+    the configuration script will find it automatically.
+
+    Returns
+    -------
+    msg : str
+        path to extracted files on success, otherwise indicates what went wrong
+        To determine success, do ``os.path.exists(msg)``
+    '''
+    _, tmp = mkstemp()
+    if not plat:
+        raise ValueError('unknown platform')
+
+    typ = download_vips(tmp, plat)
+    if not typ:
+        return ''
+    if not typ == 'tar.gz':
+        return 'expecting to download tar.gz, not %s' % str(typ)
+    return unpack_targz(tmp)
+
+
+def unpack_targz(fname):
+    target = os.path.join(os.path.dirname(__file__), 'tmp')
+    if not os.path.exists(target):
+        os.mkdir(target)
+    with tarfile.open(fname, 'r') as zf:
+        # Strip common prefix from paths when unpacking
+        prefix = os.path.commonpath(zf.getnames())
+        extract_tarfile_to(zf, target, prefix)
+        return target
+
+
+def extract_tarfile_to(tarfileobj, target_path, archive_path):
+    """Extract TarFile contents under archive_path/ to target_path/"""
+
+    target_path = os.path.abspath(target_path)
+
+    def get_members():
+        for member in tarfileobj.getmembers():
+            if archive_path:
+                norm_path = os.path.normpath(member.name)
+                if norm_path.startswith(archive_path + os.path.sep):
+                    member.name = norm_path[len(archive_path) + 1:]
+                else:
+                    continue
+
+            dst_path = os.path.abspath(os.path.join(target_path, member.name))
+            if os.path.commonpath([target_path, dst_path]) != target_path:
+                # Path not under target_path, probably contains ../
+                continue
+
+            yield member
+
+    tarfileobj.extractall(target_path, members=get_members())
+
+
+def test_setup(plats):
+    '''
+    Make sure all the downloadable files needed for wheel building
+    exist and can be opened
+    '''
+
+    errs = []
+    for plat in plats:
+        osname, _ = plat.split("-")
+        if plat not in plats:
+            continue
+        target = None
+        try:
+            try:
+                target = setup_vips(plat)
+            except Exception as e:
+                print(f'Could not setup {plat}')
+                print(e)
+                errs.append(e)
+                continue
+            if not target:
+                raise RuntimeError(f'Could not setup {plat}')
+            print('success with', plat)
+            files = [glob.glob(os.path.join(target, "lib", e))
+                     for e in ['*.so', '*.dll', '*.dylib']]
+            if not files:
+                raise RuntimeError("No files unpacked!")
+        finally:
+            if target:
+                if os.path.isfile(target):
+                    os.unlink(target)
+                else:
+                    shutil.rmtree(target)
+    if errs:
+        raise errs[0]
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description='Download and extract an libvips library for this '
+                    'architecture')
+    parser.add_argument('--test', nargs='*', default=None,
+                        help='Test different architectures. "all", or any of '
+                             f'{SUPPORTED_PLATFORMS}')
+    args = parser.parse_args()
+    if args.test is None:
+        print(setup_vips())
+    else:
+        if len(args.test) == 0 or 'all' in args.test:
+            test_setup(SUPPORTED_PLATFORMS)
+        else:
+            test_setup(args.test)

pyproject.toml

@@ -72,6 +72,42 @@ exclude = [
     "tests*",
 ]
 
+[tool.cibuildwheel]
+# Only build CPython 3.7, because we have portable abi3 wheels.
+# Windows arm64 is only available on 3.9 and later, Apple Silicon on
+# 3.8 and later.
+build = [
+    "cp37-*",
+    "cp39-win_arm64",
+    "cp38-macosx_arm64"
+]
+#build-frontend = "build"
+build-verbosity = "3"
+before-build = "bash {project}/cibw_before_build.sh"
+test-command = "python -c 'import pyvips; print(pyvips.API_mode)'"
+environment-pass = ["RUNNER_OS"]
+
+[tool.cibuildwheel.environment]
+PKG_CONFIG_PATH = "./tmp/lib/pkgconfig"
+LD_LIBRARY_PATH = "$PWD/tmp/lib"
+DYLD_LIBRARY_PATH = "$PWD/tmp/lib"
+
+[tool.cibuildwheel.linux]
+archs = "x86_64 aarch64"
+manylinux-x86_64-image = "manylinux_2_28"
+manylinux-aarch64-image = "manylinux_2_28"
+musllinux-x86_64-image = "musllinux_1_2"
+musllinux-aarch64-image = "musllinux_1_2"
+
+[tool.cibuildwheel.macos]
+# arm64 temporary removed
+archs = "x86_64"
+test-skip = "*-macosx_arm64"
+
+[tool.cibuildwheel.windows]
+archs = "AMD64 ARM64 x86"
+test-skip = "*-win_arm64"
+
 [project.optional-dependencies]
 # All the following are used for our own testing
 tox = ["tox"]

setup.py

@@ -2,6 +2,7 @@
 
 from os import path
 from setuptools import setup
+from wheel.bdist_wheel import bdist_wheel
 
 base_dir = path.dirname(__file__)
 src_dir = path.join(base_dir, 'pyvips')
@@ -15,8 +16,23 @@
 # API mode requires a working C compiler plus all the libvips headers whereas
 # ABI only needs the libvips shared library to be on the system
 
+
+class bdist_wheel_abi3(bdist_wheel):
+    def get_tag(self):
+        python, abi, plat = super().get_tag()
+
+        if python.startswith('cp'):
+            # on CPython, our wheels are abi3 and compatible back to 3.7
+            return 'cp37', 'abi3', plat
+
+        return python, abi, plat
+
+
 try:
-    setup(cffi_modules=['pyvips/pyvips_build.py:ffibuilder'])
+    setup(
+        cffi_modules=['pyvips/pyvips_build.py:ffibuilder'],
+        cmdclass={'bdist_wheel': bdist_wheel_abi3},
+    )
 except Exception as e:
     print('Falling back to ABI mode. Details: {0}'.format(e))
     setup()