Merge pull request 'feat: add support of zip source archives' (#168) from sisungo/semios-packages:zipsupport into master

Reviewed-on: semios/semios-packages#168

hooks/autostrip.py

@@ -1,6 +1,8 @@
 import os
 import stat
 import subprocess
+import tempfile
+import shutil
 
 from lib import common
 
@@ -13,6 +15,8 @@ def _is_elf(filepath):
 
 
 def autostrip(composer):
+    processed_inodes = {}
+
     for ent in common.treedir(composer.workdir):
         if not ent.is_file(follow_symlinks=False):
             continue
@@ -20,7 +24,36 @@ def autostrip(composer):
             continue
         if not _is_elf(ent.path):
             continue
-        oldperm = stat.S_IMODE(os.stat(ent.path).st_mode)
-        os.chmod(ent.path, 0o777)
-        subprocess.run([strip, ent.path], check=True)
-        os.chmod(ent.path, oldperm)
+
+        stat_info = os.stat(ent.path)
+        inode_key = (stat_info.st_dev, stat_info.st_ino)
+        if inode_key in processed_inodes:
+            continue
+        processed_inodes[inode_key] = True
+
+        oldperm = stat.S_IMODE(stat_info.st_mode)
+
+        try:
+            with tempfile.NamedTemporaryFile(mode='wb', delete=False) as tmp_file:
+                tmp_path = tmp_file.name
+                shutil.copy2(ent.path, tmp_path)
+                os.chmod(tmp_path, 0o777)
+
+            subprocess.run([strip, tmp_path], check=True)
+
+            with open(tmp_path, 'rb') as tmp_file:
+                stripped_data = tmp_file.read()
+
+            with open(ent.path, 'r+b') as original_file:
+                original_file.truncate(0)
+                original_file.write(stripped_data)
+                original_file.flush()
+                os.fsync(original_file.fileno())
+
+            os.chmod(ent.path, oldperm)
+            os.unlink(tmp_path)
+
+        except Exception as e:
+            if 'tmp_path' in locals() and os.path.exists(tmp_path):
+                os.unlink(tmp_path)
+            raise RuntimeError(f"Failed to strip {ent.path}: {e}") from e

lib/arch.py

@@ -4,16 +4,16 @@ from . import packie
 
 all: dict[str, dict[str, str]] = {
     "aarch64-semios-linux": {
-        "CC": "clang -target aarch64-semios-linux-musl",
-        "CXX": "clang++ -target aarch64-semios-linux-musl",
+        "CC": "clang -target aarch64-unknown-linux-musl",
+        "CXX": "clang++ -target aarch64-unknown-linux-musl",
     },
     "x86_64-semios-linux": {
-        "CC": "clang -target x86_64-semios-linux-musl",
-        "CXX": "clang++ -target x86_64-semios-linux-musl",
+        "CC": "clang -target x86_64-unknown-linux-musl",
+        "CXX": "clang++ -target x86_64-unknown-linux-musl",
     },
     "riscv64-semios-linux": {
-        "CC": "clang -target riscv64-semios-linux-musl",
-        "CXX": "clang++ -target riscv64-semios-linux-musl",
+        "CC": "clang -target riscv64-unknown-linux-musl",
+        "CXX": "clang++ -target riscv64-unknown-linux-musl",
     },
 }
 _target: str = packie.host_arch()

lib/atombuild.py

@@ -13,8 +13,11 @@ BASE_REQUIREMENTS = [
     # Packages for running `semios-packages` scripts
     "python",
     "packie",
-    "patch",
     "zstd",
+    "libarchive-tools",
+    # Packages for building packages that require patching
+    "patch",
+    # Packages for downloading various formats of tarballs
     "libz-ng",
     "liblzma",
     "libbzip2",
@@ -92,32 +95,20 @@ def enter_build_env(root: str, cmd: list[str], chdir: str = "/"):
     subprocess.run(
         [
             "bwrap",
-            "--bind",
-            root,
-            "/",
+            "--bind", root, "/",
             "--unshare-pid",
             "--unshare-user",
-            "--uid",
-            "0",
-            "--gid",
-            "0",
-            "--dev",
-            "/dev",
-            "--bind",
-            "/sys",
-            "/sys",
-            "--proc",
-            "/proc",
+            "--uid", "0",
+            "--gid", "0",
+            "--dev", "/dev",
+            "--bind", "/sys", "/sys",
+            "--proc", "/proc",
             "--clearenv",
-            "--setenv",
-            "HOME",
-            "/home/root",
-            "--setenv",
-            "PATH",
-            "/bin",
-            "--setenv",
-            "SEMIOS_PKGBUILD_IN_ATOMIC",
-            "1",
+            "--setenv", "HOME", "/home/root",
+            "--setenv", "PATH", "/bin",
+            "--setenv", "SEMIOS_PKGBUILD_IN_ATOMIC", "1",
+            "--setenv", "USER", "root",
+            "--setenv", "LOGNAME", "root",
             "--chdir",
             chdir,
         ]

lib/cpp.py

@@ -117,6 +117,7 @@ def _is_dev_file(ent) -> bool:
         or ent.name.endswith(".la")
         or ent.name.endswith(".pc")
         or ent.name.endswith(".h")
+        or ent.name.endswith(".hpp")
         or ent.name.endswith(".cmake")
         or ent.name.endswith(".inc")
         or ("Makefile" in ent.name)

lib/pkgcomposer.py

@@ -4,17 +4,10 @@ import shutil
 import subprocess
 import sys
 
-if sys.version_info >= (3, 14):
-    _archive_format = "zstdtar"
-    _archive_suffix = ".tar.zst"
-else:
-    _archive_format = "gztar"
-    _archive_suffix = ".tar.gz"
-
 
 class PkgComposer:
     def __init__(self, pkgfile: str):
-        self.pkgfile = pkgfile
+        self.pkgfile = os.path.abspath(pkgfile)
 
     def setworkdir(self, workdir: str):
         self.workdir = workdir
@@ -33,8 +26,16 @@ class PkgComposer:
                 f.write("\n")
 
     def compose(self):
-        shutil.make_archive(self.pkgfile, _archive_format, self.workdir)
-        shutil.move(self.pkgfile + _archive_suffix, self.pkgfile)
+        subprocess.run(
+            [
+                "tar", "cpf", f"{self.pkgfile}.tar",
+                "-C", self.workdir,
+                "--numeric-owner",
+                ".",
+            ],
+            check=True,
+        )
+        subprocess.run(["zstd", "--rm", "-f", f"{self.pkgfile}.tar", "-o", self.pkgfile], check=True)
 
     def makedir(self, name: str):
         os.mkdir(f"{self.workdir}/bundle/{name}")

lib/python.py

@@ -0,0 +1,62 @@
+import os
+import shutil
+import subprocess
+
+from . import make, arch
+
+
+PY3_VERSION = "3.14"
+
+
+def build():
+    subprocess.run(["python3", "-m", "build", "."], check=True)
+
+def install():
+    if os.path.exists("../pyinstall"):
+        shutil.rmtree("../pyinstall")
+    for i in os.listdir("dist"):
+        if not i.endswith(".whl"):
+            continue
+        subprocess.run(["python3", "-m", "installer", f"--destdir=../pyinstall", f"dist/{i}"], check=True)
+    os.chdir("../pyinstall")
+    while len(os.listdir(".")) == 1:
+        os.chdir(os.listdir(".")[0])
+
+def version(s: str):
+    make.version(f"{s}+python{PY3_VERSION}")
+
+def _depcommon(s: str):
+    if s == "python":
+        return f"python (~{PY3_VERSION}.0) @{arch.get_target()}"
+    elif ".py" in s:
+        if ")" in s:
+            return s.replace(")", f", +python{PY3_VERSION})")
+        else:
+            return s.replace(".py", f".py (+python{PY3_VERSION})")
+    else:
+        return None
+
+def builddep(s: str):
+    if s == "$shortcut":
+        builddep("python")
+        builddep("build.py")
+        builddep("installer.py")
+    elif _depcommon(s):
+        make.builddep(_depcommon(s))
+    else:
+        raise RuntimeError("unrecognized python dep")
+
+def dep(s: str):
+    if _depcommon(s):
+        make.dep(_depcommon(s))
+    else:
+        raise RuntimeError("unrecognized python dep")
+
+def _auto_pack(composer):
+    base = f"lib/python{PY3_VERSION}/site-packages"
+    composer.makedirs(base)
+    for i in os.listdir("."):
+        composer.add_dir(i, f"{base}/{i}")
+
+def use_auto_pack():
+    make.on_pack(_auto_pack)

lib/sources.py

@@ -4,6 +4,7 @@ import sys
 import tarfile
 import urllib.parse
 import urllib.request
+import zipfile
 
 
 def filename(url):
@@ -128,6 +129,25 @@ def tar_common_root(tar: tarfile.TarFile) -> str | None:
     return root
 
 
+def zip_common_root(zip_file: zipfile.ZipFile) -> str | None:
+    members = zip_file.infolist()
+    if not members:
+        return None
+    root = None
+    for member in members:
+        path = member.filename.replace('\\', '/').lstrip('./').lstrip('/')
+        if not path:
+            continue
+        this_root = path.split('/')[0]
+        if this_root in ('', '.', '..'):
+            continue
+        if root is None:
+            root = this_root
+        elif this_root != root:
+            return None
+    return root
+
+
 def sources_dir_name() -> str:
     with open("./target/sources.tag") as file:
         return file.read()
@@ -139,16 +159,28 @@ def source_url(url: str):
     download_path = f"./target/{filename(url)}"
     download(url, download_path)
 
-    with tarfile.open(download_path, "r") as tar:
-        common_root = tar_common_root(tar)
-        extract_dir = "./target"
-        sources_dir = f"./target/{common_root}"
-        if common_root in ["", ".", "None"]:
-            extract_dir = "./target/sources"
-            sources_dir = "./target/sources"
-        tar.extractall(extract_dir)
-        with open("./target/sources.tag", "wt+") as file:
-            file.write(sources_dir)
+    if download_path.endswith(".zip"):
+        with zipfile.ZipFile(download_path, "r") as zip:
+            common_root = zip_common_root(zip)
+    else:
+        with tarfile.open(download_path, "r") as tar:
+            common_root = tar_common_root(tar)
+
+    extract_dir = "./target"
+    sources_dir = f"./target/{common_root}"
+    if common_root in ["", ".", None]:
+        extract_dir = "./target/sources"
+        sources_dir = "./target/sources"
+
+    if download_path.endswith(".zip"):
+        with zipfile.ZipFile(download_path, "r") as zip:
+            zip.extractall(extract_dir)
+    else:
+        with tarfile.open(download_path, "r") as tar:
+            tar.extractall(extract_dir)
+
+    with open("./target/sources.tag", "wt+") as file:
+        file.write(sources_dir)
 
 
 def _git_repo_name(url: str) -> str:

packages/Mako-py/pkgbuild.py

@@ -0,0 +1,29 @@
+import os
+
+from lib.make import *
+from lib import python
+
+
+python.version("1.4.1")
+description("A super-fast templating language that borrows the best ideas from the existing templating languages.")
+maintainer("sisungo <[email protected]>")
+source_url("https://files.pythonhosted.org/packages/2a/12/b5fa2353e2754cd67fb9f83793fa48ff42c213a5da7e719869d2301f6ab8/mako-1.4.1.tar.gz")
+
+python.builddep("$shortcut")
+python.builddep("MarkupSafe.py")
+
+
+def build():
+    python.build()
+    python.install()
+    os.chdir("lib")
+    while len(os.listdir(".")) == 1:
+        os.chdir(os.listdir(".")[0])
+
+on_build(build)
+
+
+package("Mako.py")
+python.dep("python")
+python.dep("MarkupSafe.py")
+python.use_auto_pack()

packages/MarkupSafe-py/pkgbuild.py

@@ -0,0 +1,22 @@
+from lib.make import *
+from lib import python
+
+
+python.version("3.0.3")
+description("Safely add untrusted strings to HTML/XML markup.")
+maintainer("sisungo <[email protected]>")
+source_url("https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz")
+
+python.builddep("$shortcut")
+
+
+def build():
+    python.build()
+    python.install()
+
+on_build(build)
+
+
+package("MarkupSafe.py")
+python.dep("python")
+python.use_auto_pack()

packages/PyYAML/pkgbuild.py

@@ -0,0 +1,22 @@
+from lib.make import *
+from lib import python
+
+
+python.version("6.0.3")
+description("YAML parser and emitter for Python")
+maintainer("sisungo <[email protected]>")
+source_url("https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz")
+
+python.builddep("$shortcut")
+
+
+def build():
+    python.build()
+    python.install()
+
+on_build(build)
+
+
+package("PyYAML.py")
+python.dep("python")
+python.use_auto_pack()

packages/SPIRV-LLVM-Translator/pkgbuild.py

@@ -0,0 +1,37 @@
+from lib.make import *
+from lib import cpp, arch
+
+
+version("22.1.5")
+description("A tool and a library for bi-directional translation between SPIR-V and LLVM IR")
+maintainer("sisungo <[email protected]>")
+source_url("https://github.com/KhronosGroup/SPIRV-LLVM-Translator/archive/refs/tags/v22.1.5.tar.gz")
+
+builddep("shortcut/cmake")
+builddep("shortcut/c++")
+builddep(f"libllvm-dev (~22.1.0) @{arch.get_target()}")
+builddep(f"libllvm-static (~22.1.0) @{arch.get_target()}")
+builddep("pkgconf")
+builddep("spirv-headers")
+
+
+def build():
+    cpp.cmake([
+        "-DLLVM_EXTERNAL_SPIRV_HEADERS_SOURCE_DIR=ON",
+        f"-DLLVM_DIR=/lib/{arch.get_target()}/cmake/llvm",
+        f"-DSPIRV-Headers_SOURCE_DIR=/lib/{arch.get_target()}",
+        f"-DCMAKE_INSTALL_PREFIX={get_prefix('libllvmspirv-dev')}",
+    ])
+    cpp.ninja()
+    cpp.ninja_install()
+
+on_build(build)
+
+
+package("libllvmspirv-dev")
+cpp.use_auto_pack(["dev"])
+
+package("llvm-spirv")
+dep("libllvm (~22.1.0) @same-arch")
+dep("libcxx @same-arch")
+cpp.use_auto_pack(["bin"])

packages/build-py/pkgbuild.py

@@ -0,0 +1,24 @@
+from lib.make import *
+from lib import python
+
+
+python.version("1.5.0")
+description("A simple, correct Python build frontend")
+maintainer("sisungo <[email protected]>")
+source_url("https://files.pythonhosted.org/packages/78/e0/df5e171f685f82f37b12e1f208064e24244911079d7b767447d1af7e0d70/build-1.5.0.tar.gz")
+
+python.builddep("$shortcut")
+python.builddep("packaging.py (>=25.0)")
+
+
+def build():
+    python.build()
+    python.install()
+
+on_build(build)
+
+
+package("build.py")
+python.dep("python")
+python.dep("packaging.py (>=25.0)")
+python.use_auto_pack()

packages/cargo/pkgbuild.py

@@ -0,0 +1,33 @@
+from lib.make import *
+from lib import rust
+
+
+version("0.98.0")
+description("The Rust package manager")
+maintainer("sisungo <[email protected]>")
+source_url("https://github.com/rust-lang/cargo/archive/refs/tags/0.98.0.tar.gz")
+
+builddep("shortcut/c")
+builddep("cargo")
+builddep("rust")
+builddep(f"aws-lc-dev @{arch.get_target()}")
+builddep(f"pkgconf @{arch.get_target()}")
+builddep(f"libz-ng-dev @{arch.get_target()}")
+builddep(f"libcurl-dev @{arch.get_target()}")
+builddep(f"libgit2-dev @{arch.get_target()}")
+builddep(f"libssh2-dev @{arch.get_target()}")
+
+
+def build():
+    rust.build()
+
+on_build(build)
+
+
+package("cargo")
+dep("libssl @same-arch")
+dep("libcrypto @same-arch")
+dep("libz @same-arch")
+dep("semi-libc @same-arch")
+recommend("rust")
+rust.use_auto_pack(["bin"])

packages/git/pkgbuild.py

@@ -1,5 +1,3 @@
-import subprocess
-
 from lib import arch, cpp
 from lib.make import *
 
@@ -10,8 +8,14 @@ source_url("https://www.kernel.org/pub/software/scm/git/git-2.55.0.tar.gz")
 
 builddep("shortcut/c")
 builddep("shortcut/autotools")
+builddep("rust")
+builddep("cargo")
 builddep(f"libintl-dev @{arch.get_target()}")
 builddep(f"libz-dev @{arch.get_target()}")
+builddep(f"aws-lc-dev @{arch.get_target()}")
+builddep(f"linux-dev @{arch.get_target()}")
+builddep("gettext")
+builddep("libarchive-tools")
 
 
 def build():

packages/installer-py/pkgbuild.py

@@ -0,0 +1,22 @@
+from lib.make import *
+from lib import python
+
+
+python.version("1.0.1")
+description("A library for installing Python wheels.")
+maintainer("sisungo <[email protected]>")
+source_url("https://files.pythonhosted.org/packages/06/fe/b9f481cf0cc867958a21338baa900357b7b7d86cac9b025948049d77923c/installer-1.0.1.tar.gz")
+
+python.builddep("$shortcut")
+
+
+def build():
+    python.build()
+    python.install()
+
+on_build(build)
+
+
+package("installer.py")
+python.dep("python")
+python.use_auto_pack()

packages/llvm-project/pkgbuild.py

@@ -28,7 +28,7 @@ srcdir = None
 packname = None
 
 def llvm_host_triple():
-    return arch.get_target().replace("-semios-linux", "-semios-linux-musl")
+    return arch.get_target().replace("-semios-linux", "-unknown-linux-musl")
 
 def cmake(sourcedir, args):
     builddir = os.getcwd() + f"/../_build_{sourcedir}"
@@ -76,7 +76,7 @@ def auto_pack_dev(composer, name, devbins=[], extdirs=["lib/cmake"]):
     composer.makedir("lib")
     for i in os.listdir("lib"):
         prefix = None
-        if os.path.isdir(i) and f"lib/{i}" in extdirs:
+        if os.path.isdir(f"lib/{i}") and f"lib/{i}" in extdirs:
             composer.add_dir(f"lib/{i}", f"lib/{i}")
         elif ".so" in i:
             prefix = get_prefix(f"lib{name}")
@@ -132,7 +132,7 @@ def build_clang():
         "-DCLANG_DEFAULT_CXX_STDLIB=libc++",
         "-DCLANG_DEFAULT_RTLIB=compiler-rt",
         "-DLLVM_INCLUDE_TESTS=OFF",
-        f"-DC_INCLUDE_DIRS=/lib/{arch.get_target()}/include",
+        f"-DC_INCLUDE_DIRS=/lib/{arch.get_target()}/include:{get_prefix('clang')}/lib/clang/{pkgver_major}/include",
     ]
     cmake("clang", import_llvm() + options)
 

packages/packaging-py/pkgbuild.py

@@ -0,0 +1,22 @@
+from lib.make import *
+from lib import python
+
+
+python.version("26.3")
+description("Core utilities for Python packages")
+maintainer("sisungo <[email protected]>")
+source_url("https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz")
+
+python.builddep("$shortcut")
+
+
+def build():
+    python.build()
+    python.install()
+
+on_build(build)
+
+
+package("packaging.py")
+python.dep("python")
+python.use_auto_pack()

packages/spirv-tools/pkgbuild.py

@@ -0,0 +1,43 @@
+import os
+import shutil
+
+from lib.make import *
+from lib import cpp, arch
+
+
+sdk_version = "1.4.357.0"
+
+version(sdk_version)
+description("an API and commands for processing SPIR-V modules")
+maintainer("sisungo <[email protected]>")
+source_url("https://github.com/KhronosGroup/SPIRV-Tools/archive/refs/tags/vulkan-sdk-1.4.357.0.tar.gz")
+
+builddep("shortcut/cmake")
+builddep("shortcut/c++")
+builddep(f"spirv-headers (={sdk_version}) @{arch.get_target()}")
+
+
+def build():
+    cpp.cmake([
+        f"-DSPIRV-Headers_SOURCE_DIR=/lib/{arch.get_target()}",
+        "-DCMAKE_CXX_FLAGS=-Wno-error=switch",
+        f"-DCMAKE_INSTALL_PREFIX={get_prefix('spirv-tools-dev')}",
+    ])
+    cpp.ninja()
+    cpp.ninja_install()
+    try:
+        os.rename("lib64", "lib")
+        os.rename("lib32", "lib")
+    except:
+        pass
+
+on_build(build)
+
+
+package("spirv-tools")
+dep("libcxx @same-arch")
+cpp.use_auto_pack(["bin"])
+
+package("spirv-tools-dev")
+dep("spirv-tools (same)")
+cpp.use_auto_pack(["lib", "dev"])

x.py

@@ -118,6 +118,8 @@ def cmd_build_package(package: str):
             "name": pkgname,
             "version": pkgver,
             "arch": pkgarch,
+            "description": pkginfo[lib.make._key_description],
+            "maintainers": pkginfo[lib.make._key_maintainers],
             "dependencies": pkginfo[lib.make._key_dependencies],
             "provides": pkginfo[lib.make._key_provides],
             "recommendations": pkginfo[lib.make._key_recommendations],