Merge pull request #208 from bgilbert/direct

bgilbert · web-flow · commit bbc7ef8747d2 · 2023-12-23T10:52:19.000-06:00
Add tool for combining macOS builds into universal build
diff --git a/.github/workflows/direct.yml b/.github/workflows/direct.yml
@@ -25,27 +25,25 @@ jobs:
           key: direct-cache
           path: subprojects/packagecache
       - name: Build
+        id: build
         run: |
           meson setup build --native-file machines/native-linux-x86_64.ini \
               -Dopenslide:werror=true -Dopenslide-java:werror=true
           meson compile -C build
-          DESTDIR=install meson install -C build
-          mkdir output
-          cp build/install/{bin/slidetool,lib64/libopenslide.so.1} output/
-          cd output
-          for f in libopenslide.so.1 slidetool; do
-              objcopy --only-keep-debug $f ${f}.debug
-              objcopy -S --add-gnu-debuglink=${f}.debug $f ${f}.new
-              mv ${f}.new $f
-          done
-          patchelf --set-rpath '$ORIGIN' slidetool
+          artifact=$(cd build/artifacts && echo openslide-bin-*-linux-x86_64.tar.xz)
+          mv "build/artifacts/$artifact" .
+          echo "artifact=$artifact" >> $GITHUB_OUTPUT
       - name: Smoke test
-        run: OPENSLIDE_DEBUG=synthetic output/slidetool prop list ''
+        run: |
+          tar xf "${{ steps.build.outputs.artifact }}"
+          OPENSLIDE_DEBUG=synthetic \
+              openslide-bin-*-linux-x86_64/bin/slidetool prop list ''
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: linux
-          path: output
+          name: ${{ steps.build.outputs.artifact }}
+          path: ${{ steps.build.outputs.artifact }}
+          compression-level: 0
   macos:
     name: macOS
     runs-on: macos-latest
@@ -68,29 +66,28 @@ jobs:
           key: direct-cache
           path: subprojects/packagecache
       - name: Build
+        id: build
         run: |
+          export OPENSLIDE_BIN_SUFFIX="$(date +%Y%m%d).local"
+          version=$(MESON_SOURCE_ROOT=$(pwd) python3 utils/get-version.py)
           for arch in x86_64 arm64; do
               meson setup $arch --cross-file machines/cross-macos-${arch}.ini \
                   -Dopenslide:werror=true -Dopenslide-java:werror=true
               meson compile -C $arch
-              DESTDIR=install meson install -C $arch
           done
-          mkdir output
-          lipo -create {x86_64,arm64}/install/lib/libopenslide.1.dylib \
-              -output output/libopenslide.1.dylib
-          lipo -create {x86_64,arm64}/install/bin/slidetool \
-              -output output/slidetool
-          cd output
-          for f in libopenslide.1.dylib slidetool; do
-              dsymutil $f
-              strip -u -r $f
-          done
-          install_name_tool -change /lib/libopenslide.1.dylib \
-              '@loader_path/libopenslide.1.dylib' slidetool
+          artifact="openslide-bin-${version}-macos-arm64-x86_64.tar.xz"
+          PYTHONPATH=. python3 utils/write-universal-bdist.py \
+              -o "$artifact" \
+              */artifacts/openslide-bin-"${version}"-macos-*.tar.xz
+          echo "artifact=$artifact" >> $GITHUB_OUTPUT
       - name: Smoke test
-        run: OPENSLIDE_DEBUG=synthetic output/slidetool prop list ''
+        run: |
+          tar xf "${{ steps.build.outputs.artifact }}"
+          OPENSLIDE_DEBUG=synthetic \
+              openslide-bin-*-macos-arm64-x86_64/bin/slidetool prop list ''
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: macos
-          path: output
+          name: ${{ steps.build.outputs.artifact }}
+          path: ${{ steps.build.outputs.artifact }}
+          compression-level: 0
diff --git a/common/archive.py b/common/archive.py
@@ -20,21 +20,36 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from collections.abc import Iterable, Iterator, Sequence
+from contextlib import ExitStack, contextmanager
+import copy
 from dataclasses import dataclass
+from functools import cached_property
+from itertools import zip_longest
 import os
 from pathlib import Path, PurePath
 import re
 import tarfile
+import tempfile
 import time
 from types import TracebackType
-from typing import BinaryIO, Self
+from typing import BinaryIO, Self, cast
 import zipfile
 
 
 @dataclass
 class Member(ABC):
     path: PurePath
 
+    @property
+    def relpath(self) -> PurePath:
+        return PurePath(*self.path.parts[1:])
+
+    def with_base(self, base: PurePath) -> Member:
+        member = copy.copy(self)
+        member.path = base / member.relpath
+        return member
+
 
 @dataclass
 class FileMember(Member):
@@ -53,7 +68,7 @@ class SymlinkMember(Member):
 
 class ArchiveWriter(ABC):
     def __init__(self, path: Path):
-        self.base = PurePath(re.sub('\\.(tar\\.xz|zip)$', '', path.name))
+        self.base = _path_base(path)
         self._members: dict[PurePath, Member] = {}
 
     def __enter__(self) -> Self:
@@ -163,3 +178,101 @@ def close(self) -> None:
             elif isinstance(member, SymlinkMember):
                 raise Exception('Symlinks not supported in Zip')
         self._zip.close()
+
+
+class ArchiveReader(ABC):
+    def __init__(self, path: Path):
+        self.base = _path_base(path)
+        self._tempdir = tempfile.TemporaryDirectory(prefix='openslide-bin-')
+        self._dir = Path(self._tempdir.name)
+
+    @classmethod
+    @contextmanager
+    def group(cls, fhs: Iterable[BinaryIO]) -> Iterator[Iterator[MemberSet]]:
+        with ExitStack() as stack:
+            readers = [
+                # mypy thinks we're initializing this ABC, not a subclass
+                stack.enter_context(cls(fh))  # type: ignore[arg-type]
+                for fh in fhs
+            ]
+            yield (MemberSet(members) for members in zip_longest(*readers))
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    @abstractmethod
+    def close(self) -> None:
+        self._tempdir.cleanup()
+
+    @abstractmethod
+    def __iter__(self) -> Iterator[Member]:
+        pass
+
+
+class TarArchiveReader(ArchiveReader):
+    def __init__(self, fh: BinaryIO):
+        super().__init__(Path(fh.name))
+        self._tar = tarfile.open(fileobj=fh)
+        if hasattr(tarfile, 'data_filter'):
+            self._tar.extraction_filter = tarfile.data_filter
+
+    def close(self) -> None:
+        self._tar.close()
+        super().close()
+
+    def __iter__(self) -> Iterator[Member]:
+        while True:
+            info = self._tar.next()
+            if info is None:
+                return
+            path = PurePath(info.name)
+            if info.type == tarfile.DIRTYPE:
+                yield DirMember(path)
+            elif info.type == tarfile.REGTYPE:
+                self._tar.extract(info, self._dir)
+                yield FileMember(path, open(self._dir / path, 'rb'))
+            elif info.type == tarfile.SYMTYPE:
+                yield SymlinkMember(path, PurePath(info.linkname))
+            else:
+                raise Exception(
+                    f'Unsupported member type: {info.type.decode()}'
+                )
+
+
+class MemberSet:
+    def __init__(self, members: Sequence[Member | None]):
+        if not all(members):
+            raise Exception('Missing member in one or more archives')
+        self.members = cast(Sequence[Member], members)
+
+    def __getitem__(self, idx: int) -> Member:
+        return self.members[idx]
+
+    def __iter__(self) -> Iterator[Member]:
+        return iter(self.members)
+
+    @property
+    def relpaths(self) -> Sequence[PurePath]:
+        return [member.relpath for member in self]
+
+    @cached_property
+    def datas(self) -> Sequence[bytes]:
+        ret = []
+        for member in self:
+            if not isinstance(member, FileMember):
+                raise Exception('Member is not a file')
+            ret.append(member.fh.read())
+            member.fh.seek(0)
+        return ret
+
+
+def _path_base(path: Path) -> PurePath:
+    return PurePath(re.sub('\\.(tar\\.xz|zip)$', '', path.name))
diff --git a/common/macos.py b/common/macos.py
@@ -0,0 +1,45 @@
+#
+# Tools for building OpenSlide and its dependencies
+#
+# Copyright (c) 2023 Benjamin Gilbert
+# All rights reserved.
+#
+# This script is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License, version 2.1,
+# as published by the Free Software Foundation.
+#
+# This script is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
+# for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this script. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+import os
+from pathlib import Path
+import subprocess
+from typing import Any
+
+
+def merge_macho(paths: Sequence[Path], outdir: Path) -> Path:
+    outpath = outdir / paths[0].name
+    args: list[str | Path] = [
+        os.environ.get('LIPO', 'lipo'),
+        '-create',
+        '-output',
+        outpath,
+    ]
+    args.extend(paths)
+    subprocess.check_call(args)
+    return outpath
+
+
+def all_equal(items: Iterable[Any]) -> bool:
+    it = iter(items)
+    first = next(it)
+    return all(i == first for i in it)
diff --git a/utils/write-universal-bdist.py b/utils/write-universal-bdist.py
