diff --git a/.ci/lib/apkbuild_linting.py b/.ci/lib/apkbuild_linting.py index c14050a91..e7635b63c 100755 --- a/.ci/lib/apkbuild_linting.py +++ b/.ci/lib/apkbuild_linting.py @@ -21,7 +21,7 @@ custom_valid_options = [ ] -def get_kconfigcheck_categories() -> list [str]: +def get_kconfigcheck_categories() -> list[str]: with open("kconfigcheck.toml", "rb") as kconfigcheck_config: kconfigcheck_data = tomllib.load(kconfigcheck_config) @@ -34,27 +34,33 @@ def get_kconfigcheck_categories() -> list [str]: else: kconfigcheck_categories.append(category_name) - return [item.replace("category:", "pmb:kconfigcheck-") for item in kconfigcheck_categories] + return [ + item.replace("category:", "pmb:kconfigcheck-") + for item in kconfigcheck_categories + ] if __name__ == "__main__": if common.commit_message_has_string("[ci:skip-apkbuild-lint]"): print("WARNING: not linting deviceinfo files ([ci:skip-apkbuild-lint])") - exit(0) + sys.exit(0) kconfigcheck_categories = get_kconfigcheck_categories() custom_valid_options += kconfigcheck_categories os.environ["CUSTOM_VALID_OPTIONS"] = " ".join(custom_valid_options) - apkbuilds = {file for file in common.get_changed_files() - if os.path.basename(file) == "APKBUILD"} + apkbuilds = { + file + for file in common.get_changed_files() + if os.path.basename(file) == "APKBUILD" + } if len(apkbuilds) < 1: print("No APKBUILDs to lint") sys.exit(0) apkbuilds_filtered = [] for apkbuild in apkbuilds: - if apkbuild.startswith("temp/") or apkbuild.startswith("cross/"): + if apkbuild.startswith(("temp/", "cross/")): print(f"NOTE: Skipping linting of {apkbuild}") continue apkbuilds_filtered.append(apkbuild) diff --git a/.ci/lib/build_changed_aports.py b/.ci/lib/build_changed_aports.py index e7cc01186..8c0cf2467 100755 --- a/.ci/lib/build_changed_aports.py +++ b/.ci/lib/build_changed_aports.py @@ -21,11 +21,20 @@ def build_strict(packages, arch: Arch): common.run_pmbootstrap(["build_init"]) # We set the timeout to 1 hour because linking the Linux kernel with # ThinLTO on our aarch64 runners takes slightly over 30 minutes - common.run_pmbootstrap(["--details-to-stdout", "--no-ccache", - "--timeout", "3600", - "build", - "--strict", "--force", - "--arch", str(arch), ] + list(packages)) + common.run_pmbootstrap( + [ + "--details-to-stdout", + "--no-ccache", + "--timeout", + "3600", + "build", + "--strict", + "--force", + "--arch", + str(arch), + ] + + list(packages) + ) if __name__ == "__main__": @@ -51,7 +60,7 @@ if __name__ == "__main__": buildable_pkgs.add(os.path.basename(path)) # To store a list of packages from extra-repos/systemd for special handling later: - systemd_pkgs = list() + systemd_pkgs = [] # Filter out packages that either: # 1. can't be built for given arch @@ -92,10 +101,16 @@ if __name__ == "__main__": # filter out packages that can't be built for arch # (Iterate over copy of `systemd_pkgs`, because we modify it in this loop) for package in systemd_pkgs.copy(): - apkbuild_path = pmb.helpers.pmaports.find(package, True, True, with_extra_repos="enabled") - apkbuild = pmb.parse._apkbuild.apkbuild(pathlib.Path(apkbuild_path, "APKBUILD")) + apkbuild_path = pmb.helpers.pmaports.find( + package, True, True, with_extra_repos="enabled" + ) + apkbuild = pmb.parse._apkbuild.apkbuild( + pathlib.Path(apkbuild_path, "APKBUILD") + ) if arch not in Arch.from_arch_field(apkbuild["arch"]): - print(f"(extra-repos/systemd) {package}: not enabled for {arch}, skipping") + print( + f"(extra-repos/systemd) {package}: not enabled for {arch}, skipping" + ) systemd_pkgs.remove(package) # No packages: skip build diff --git a/.ci/lib/check_changed_versions.py b/.ci/lib/check_changed_versions.py index 55b141aa4..5a8a00836 100755 --- a/.ci/lib/check_changed_versions.py +++ b/.ci/lib/check_changed_versions.py @@ -26,8 +26,7 @@ def get_package_contents(package, revision, check=True): stderr = subprocess.DEVNULL # Run something like "git show upstream/main:main/hello-world/APKBUILD" - apkbuild_content = common.run_git(["show", revision + ":" + package], check, - stderr) + apkbuild_content = common.run_git(["show", revision + ":" + package], check, stderr) if not apkbuild_content: return None @@ -36,15 +35,14 @@ def get_package_contents(package, revision, check=True): with tempfile.TemporaryDirectory() as tempdir: with open(tempdir + "/APKBUILD", "w", encoding="utf-8") as handle: handle.write(apkbuild_content) - parsed = pmb.parse.apkbuild(pathlib.Path(tempdir, "APKBUILD"), - False, False) + parsed = pmb.parse.apkbuild(pathlib.Path(tempdir, "APKBUILD"), False, False) return parsed def get_package_version(package, revision, check=True): - """ returns version in the format "{pkgver}-r{pkgrel}", or None if no - matching package is found """ + """returns version in the format "{pkgver}-r{pkgrel}", or None if no + matching package is found""" parsed = get_package_contents(package, revision, check) if parsed is None: return None @@ -52,7 +50,7 @@ def get_package_version(package, revision, check=True): def version_compare_operator(result): - """ :param result: return value from pmb.parse.version.compare() """ + """:param result: return value from pmb.parse.version.compare()""" if result == -1: return "<" elif result == 0: @@ -88,7 +86,7 @@ def exit_with_error_message(): print(" latest commit message, then force push.") print() print("Thank you and sorry for the inconvenience.") - exit(1) + sys.exit(1) def check_versions(packages): @@ -101,14 +99,21 @@ def check_versions(packages): # desired here, since we already know what packages changed, and really # want to check if the version was increased towards *current* upstream # branch HEAD. - common.run_git(["remote", "add", "upstream", - "https://gitlab.postmarketos.org/postmarketOS/pmaports.git"], False) + common.run_git( + [ + "remote", + "add", + "upstream", + "https://gitlab.postmarketos.org/postmarketOS/pmaports.git", + ], + False, + ) common.run_git(["fetch", "-q", "upstream"]) commit = f"upstream/{common.get_upstream_branch()}" - if common.run_git(["rev-parse", "HEAD"]) == common.run_git(["rev-parse", - commit]): - print(f"NOTE: {commit} is on same commit as HEAD, comparing" - " HEAD against HEAD~1.") + if common.run_git(["rev-parse", "HEAD"]) == common.run_git(["rev-parse", commit]): + print( + f"NOTE: {commit} is on same commit as HEAD, comparing HEAD against HEAD~1." + ) commit = "HEAD~1" # Exclude linux-next from pkgrel checks, as it is updated automatically. @@ -124,28 +129,35 @@ def check_versions(packages): head = get_package_version(package, "HEAD") upstream = get_package_version(package, commit, False) if not upstream: - if head.rpartition('r')[2] != "0": + if head.rpartition("r")[2] != "0": print(f"- {package}: {head} (HEAD) (new package) [ERROR]") error = True else: print(f"- {package}: {head} (HEAD) (new package)") continue - is_device_pkg = os.path.basename(package).startswith('device-') + is_device_pkg = os.path.basename(package).startswith("device-") if is_device_pkg: # Check pkgver follows expected format for device packages - pkgver = head.rpartition('-r')[0] + pkgver = head.rpartition("-r")[0] if not pkgver.isdigit(): - print(f" - {package}: invalid pkgver \"{pkgver}\"" - "See: https://docs.postmarketos.org/pmaports/main/packaging-guidelines.html#package-versioning-pkgver-pkgrel") + print( + f' - {package}: invalid pkgver "{pkgver}"' + "See: https://docs.postmarketos.org/pmaports/main/packaging-guidelines.html#package-versioning-pkgver-pkgrel" + ) error = True # Check that pkgrel was reset to 0 when pkgver was changed head_parsed = get_package_contents(package, "HEAD", False) upstream_parsed = get_package_contents(package, commit, False) - if head_parsed["pkgver"] != upstream_parsed["pkgver"] and head_parsed["pkgrel"] != "0": - print(f" - {package}: pkgrel should be 0 when pkgver changes." - "See: https://docs.postmarketos.org/pmaports/main/packaging-guidelines.html#package-versioning-pkgver-pkgrel") + if ( + head_parsed["pkgver"] != upstream_parsed["pkgver"] + and head_parsed["pkgrel"] != "0" + ): + print( + f" - {package}: pkgrel should be 0 when pkgver changes." + "See: https://docs.postmarketos.org/pmaports/main/packaging-guidelines.html#package-versioning-pkgver-pkgrel" + ) error = True # Compare head and upstream versions @@ -169,15 +181,16 @@ if __name__ == "__main__": packages = common.get_changed_packages() if len(packages) == 0: print("no aports changed in this branch") - exit(0) + sys.exit(0) print(f"Changed packages: {packages}") # Potentially skip this check if common.commit_message_has_string("[ci:skip-vercheck]"): - print("WARNING: not checking for changed package versions" - " ([ci:skip-vercheck])!") - exit(0) + print( + "WARNING: not checking for changed package versions ([ci:skip-vercheck])!" + ) + sys.exit(0) # Initialize args (so we can use pmbootstrap's APKBUILD parsing) sys.argv = ["pmbootstrap.py", "chroot"] @@ -187,7 +200,6 @@ if __name__ == "__main__": # pmb.helpers.logging.init(args) pmb.helpers.logging.init(context.log, args.verbose, context.details_to_stdout) - # Verify package versions print("checking changed package versions...") check_versions(packages) diff --git a/.ci/lib/check_devices_in_wiki.py b/.ci/lib/check_devices_in_wiki.py index a418e34af..00f083642 100755 --- a/.ci/lib/check_devices_in_wiki.py +++ b/.ci/lib/check_devices_in_wiki.py @@ -22,7 +22,7 @@ def get_devices() -> list[str]: # devices that have a working mainline kernel. Those are usually # unmaintained and therefore might not appear in the wiki. However, # the main device should be documented (remove the -downstream suffix). - device = device.removesuffix('-downstream') + device = device.removesuffix("-downstream") ret.append(device) return sorted(ret) @@ -30,10 +30,10 @@ def get_devices() -> list[str]: def get_wiki_devices_html(path: Path | None) -> dict[str, str]: """:param path: to a local file with the saved content of the devices wiki - page or None to download a fresh copy - :returns: HTML of the page, split into booting and not booting: - {"booting": "\n\n

These..."}""" + page or None to download a fresh copy + :returns: HTML of the page, split into booting and not booting: + {"booting": "\n\n

These..."}""" content = "" if path: # Read file @@ -46,7 +46,7 @@ def get_wiki_devices_html(path: Path | None) -> dict[str, str]: content = f.read().decode("utf-8") # Split into booting and not booting - split = content.split("") + split = content.split('') if len(split) != 2: print("*** Failed to parse wiki page") @@ -65,16 +65,18 @@ def get_wiki_renamed_devices_html() -> str: def check_device(device: str, html: dict[str, str], is_booting: bool) -> bool: """:param is_booting: require the device to be in the booting section, not - just anywhere in the page (i.e. in the not booting - table). - :returns: True when the device is in the appropriate section.""" + just anywhere in the page (i.e. in the not booting + table). + :returns: True when the device is in the appropriate section.""" if device in html["booting"]: return True if device in html["not_booting"]: if is_booting: - print(device + ": still in 'not booting' section (if this is a" - " merge request, your device should be in the booting" - " section already)") + print( + device + ": still in 'not booting' section (if this is a" + " merge request, your device should be in the booting" + " section already)" + ) return False return True if device in html["renamed"]: @@ -88,17 +90,27 @@ def check_device(device: str, html: dict[str, str], is_booting: bool) -> bool: def main() -> int: # Parse arguments parser = argparse.ArgumentParser() - parser.add_argument("--booting", help="devices must be in the upper table," - " being in the 'not booting' table below is not" - " enough (all devices in pmaports main and stable" - " branches and should be in the upper table)", - action="store_true") - parser.add_argument("--path", help="instead of downloading the devices" - " page from the wiki, use a local HTML file", - type=Path, default=None) - parser.add_argument("--all", help="check all devices (default: check only" - " devices changed from branch point)", - action="store_true") + parser.add_argument( + "--booting", + help="devices must be in the upper table," + " being in the 'not booting' table below is not" + " enough (all devices in pmaports main and stable" + " branches and should be in the upper table)", + action="store_true", + ) + parser.add_argument( + "--path", + help="instead of downloading the devices" + " page from the wiki, use a local HTML file", + type=Path, + default=None, + ) + parser.add_argument( + "--all", + help="check all devices (default: check only" + " devices changed from branch point)", + action="store_true", + ) args = parser.parse_args() html = get_wiki_devices_html(args.path) diff --git a/.ci/lib/common.py b/.ci/lib/common.py index 8d9771f74..31246c63f 100755 --- a/.ci/lib/common.py +++ b/.ci/lib/common.py @@ -16,7 +16,7 @@ def get_pmaports_dir(): def run_git(parameters, check=True, stderr=None): - """ Run git in the pmaports dir and return the output """ + """Run git in the pmaports dir and return the output""" cmd = ["git", "-C", get_pmaports_dir()] + parameters try: return subprocess.check_output(cmd, stderr=stderr).decode() @@ -34,23 +34,25 @@ def commit_message_has_string(needle): def all_committed_by_merge_bot(): base_commit = get_base_commit() - commit_emails = run_git(["log", "--pretty=format:%ce", f"{base_commit}..HEAD"]).splitlines() + commit_emails = run_git( + ["log", "--pretty=format:%ce", f"{base_commit}..HEAD"] + ).splitlines() bot_email = os.getenv("PMOS_MERGE_BOT_EMAIL", "merge-bot@postmarketos.org") return all(bot_email == ce for ce in commit_emails) def run_pmbootstrap(parameters): - """ Run pmbootstrap with the pmaports dir as --aports """ + """Run pmbootstrap with the pmaports dir as --aports""" cmd = ["pmbootstrap", "--aports", get_pmaports_dir()] + parameters subprocess.run(cmd, text=True, check=True) @cache def get_upstream_branch(): - """ Use pmaports.cfg from current branch (e.g. "v20.05_fix-ci") and - channels.cfg from main to retrieve the upstream branch. + """Use pmaports.cfg from current branch (e.g. "v20.05_fix-ci") and + channels.cfg from main to retrieve the upstream branch. - :returns: branch name, e.g. "v20.05" """ + :returns: branch name, e.g. "v20.05" """ # Prefer gitlab CI target branch name if it's set (i.e. running in gitlab CI) if target_branch := os.environ.get("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"): @@ -68,10 +70,11 @@ def get_upstream_branch(): channels_cfg_str = run_git(["show", "upstream/main:channels.cfg"]) channels_cfg = configparser.ConfigParser() channels_cfg.read_string(channels_cfg_str) - assert channel in channels_cfg, \ - f"Channel '{channel}' from pmaports.cfg in your branch is unknown." \ - " This appears to be an old branch, consider recreating your change" \ + assert channel in channels_cfg, ( + f"Channel '{channel}' from pmaports.cfg in your branch is unknown." + " This appears to be an old branch, consider recreating your change" " on top of master." + ) return channels_cfg[channel]["branch_pmaports"] @@ -86,8 +89,15 @@ def get_base_commit() -> str: return commit # Add a remote pointing to postmarketOS/pmaports - run_git(["remote", "add", "upstream", - "https://gitlab.postmarketos.org/postmarketOS/pmaports.git"], False) + run_git( + [ + "remote", + "add", + "upstream", + "https://gitlab.postmarketos.org/postmarketOS/pmaports.git", + ], + False, + ) run_git(["fetch", "-q", "upstream"]) branch_upstream = f"upstream/{get_upstream_branch()}" @@ -109,10 +119,10 @@ def get_base_commit() -> str: def get_changed_files(removed=False): - """ Get all changed files and print them, as well as the branch and the - commit that was used for the diff. - :param removed: also return removed files (default: True) - :returns: set of changed files + """Get all changed files and print them, as well as the branch and the + commit that was used for the diff. + :param removed: also return removed files (default: True) + :returns: set of changed files """ commit = get_base_commit() @@ -131,7 +141,9 @@ def get_changed_files(removed=False): return ret -def get_changed_packages(skip_archived: bool = False, keep_dir: bool = False) -> set[str]: +def get_changed_packages( + skip_archived: bool = False, keep_dir: bool = False +) -> set[str]: ret = set() for file in get_changed_files(removed=True): dirname, filename = os.path.split(file) @@ -141,11 +153,12 @@ def get_changed_packages(skip_archived: bool = False, keep_dir: bool = False) -> # * path with a dot (e.g. .ci/, device/.shared-patches/) # * documentation if ( - not dirname - or file.startswith(".") - or "/." in file - or skip_archived and dirname.startswith("device/archived") - or dirname == "docs" + not dirname + or file.startswith(".") + or "/." in file + or skip_archived + and dirname.startswith("device/archived") + or dirname == "docs" ): continue @@ -185,6 +198,7 @@ def get_changed_kernels(skip_archived: bool = False): ret += [pkgname] return ret + def get_changed_devices(skip_archived: bool = False): ret = [] for pkgname in get_changed_packages(skip_archived): diff --git a/.ci/lib/deviceinfo_linting.py b/.ci/lib/deviceinfo_linting.py index a8bf85e64..5174d8a92 100755 --- a/.ci/lib/deviceinfo_linting.py +++ b/.ci/lib/deviceinfo_linting.py @@ -12,7 +12,7 @@ import common if __name__ == "__main__": if common.commit_message_has_string("[ci:skip-dint]"): print("WARNING: not linting deviceinfo files ([ci:skip-dint])") - exit(0) + sys.exit(0) if "deviceinfo_schema.toml" in common.get_changed_files(): print("deviceinfo_schema.toml changed -> checking all deviceinfo files") @@ -20,8 +20,11 @@ if __name__ == "__main__": deviceinfo_files = set(glob.glob(pmaports_dir + "/device/*/*/deviceinfo")) else: # only lint deviceinfo files in the devices repo - deviceinfo_files = {file for file in common.get_changed_files() - if os.path.basename(file) == "deviceinfo" and file.startswith("device/")} + deviceinfo_files = { + file + for file in common.get_changed_files() + if os.path.basename(file) == "deviceinfo" and file.startswith("device/") + } try: subprocess.run(["dint", "check", *deviceinfo_files], text=True, check=True) diff --git a/.ci/lib/generate_build_jobs.py b/.ci/lib/generate_build_jobs.py index dce5bf3e9..a41b4894c 100755 --- a/.ci/lib/generate_build_jobs.py +++ b/.ci/lib/generate_build_jobs.py @@ -38,7 +38,7 @@ class Device: fragment_path = self.full_path / "gitlab-ci.yml.j2" try: fragment_tmpl = fragment_path.read_text() - except Exception: + except FileNotFoundError: return None return Template(fragment_tmpl).render(device=self) @@ -47,8 +47,10 @@ class Device: def pmaports_path(self) -> Path: # Find the root of pmaports pmaports_root = self.full_path.parent - while (not (pmaports_root / "pmaports.cfg").exists() and - pmaports_root != pmaports_root.root): + while ( + not (pmaports_root / "pmaports.cfg").exists() + and pmaports_root != pmaports_root.root + ): pmaports_root = pmaports_root.parent assert pmaports_root != pmaports_root.root @@ -60,11 +62,11 @@ class Device: @property def pkgname(self) -> str: - return self.apkbuild['pkgname'] + return self.apkbuild["pkgname"] @property def arch(self) -> Arch: - return Arch(self.apkbuild['arch'][0]) + return Arch(self.apkbuild["arch"][0]) @property def testing_dependencies(self) -> set[str]: @@ -88,7 +90,7 @@ class Device: kernels = [] subpackage_prefix = f"device-{self.codename}-kernel-" - for subpkgname in self.apkbuild.get('subpackages', []): + for subpkgname in self.apkbuild.get("subpackages", []): if not subpkgname.startswith(subpackage_prefix): continue kernel_name = subpkgname.removeprefix(subpackage_prefix) @@ -129,7 +131,7 @@ class Device: continue supported_devices[dev.pmaports_path] = dev - except Exception: + except FileNotFoundError: traceback.print_exc() return supported_devices @@ -138,17 +140,23 @@ class ArchTagSet(set): def update(self, iterable): supported_arches = Arch.supported_binary() # This ignores things like !armv7, that could be a follow-up optimization - if 'noarch' in iterable or 'all' in iterable: + if "noarch" in iterable or "all" in iterable: iterable = [arch for arch in supported_arches] - super().update([Arch(arch) for arch in iterable if Arch(arch) in supported_arches]) + super().update( + [Arch(arch) for arch in iterable if Arch(arch) in supported_arches] + ) if __name__ == "__main__": parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument("--template", default=".ci/build-jobs.yaml.j2", help="Jinja2 input template") - parser.add_argument("--output", default=".ci/build-jobs.yaml", help="output pipeline") + parser.add_argument( + "--template", default=".ci/build-jobs.yaml.j2", help="Jinja2 input template" + ) + parser.add_argument( + "--output", default=".ci/build-jobs.yaml", help="output pipeline" + ) args = parser.parse_args() # pmb logging has to be initialized for later pmb commands to work, setting @@ -179,13 +187,13 @@ if __name__ == "__main__": if path.name != "APKBUILD": continue apkbuild = pmb.parse.apkbuild(path) - packages_modified.add(apkbuild['pkgname']) + packages_modified.add(apkbuild["pkgname"]) archs.update(apkbuild["arch"]) # Add all the devices found in CI that depend on the package that got # modified for device in supported_devices.values(): - if apkbuild['pkgname'] in device.dependencies: + if apkbuild["pkgname"] in device.dependencies: devices_under_test.add(device) if common.commit_message_has_string("[ci:skip-build]"): diff --git a/.ci/lib/kconfig.py b/.ci/lib/kconfig.py index e6fee1c71..0105df783 100755 --- a/.ci/lib/kconfig.py +++ b/.ci/lib/kconfig.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import subprocess +import sys # Same dir import common @@ -12,10 +13,11 @@ def check_kconfig(pkgnames): last_failed = None for i in range(len(pkgnames)): pkgname = pkgnames[i] - print(f" ({i+1}/{len(pkgnames)}) {pkgname}") + print(f" ({i + 1}/{len(pkgnames)}) {pkgname}") - p = subprocess.run(["pmbootstrap", "kconfig", "check", "--keep-going", pkgname], - check=False) + p = subprocess.run( + ["pmbootstrap", "kconfig", "check", "--keep-going", pkgname], check=False + ) if p.returncode: last_failed = pkgname @@ -32,8 +34,10 @@ def show_error(last_failed): print() print("---") print() - print("Please adjust your kernel config. This is required for getting your" - " patch merged.") + print( + "Please adjust your kernel config. This is required for getting your" + " patch merged." + ) print() print("Edit your kernel config:") print(f" pmbootstrap kconfig edit {last_failed}") @@ -76,21 +80,21 @@ if __name__ == "__main__": print("kconfigcheck.toml changed -> checking all kernels") if not check_kconfig_all(): show_error_all() - exit(1) - exit(0) + sys.exit(1) + sys.exit(0) pkgnames = common.get_changed_kernels(skip_archived=True) print(f"Changed kernels: {pkgnames}") if len(pkgnames) == 0: print("No kernels changed in this branch") - exit(0) + sys.exit(0) elif common.commit_message_has_string("[ci:skip-kconfigcheck]"): print("WARNING: not checking kernel configs ([ci:skip-kconfigcheck])") - exit(0) + sys.exit(0) last_failed = check_kconfig(pkgnames) if last_failed: show_error(last_failed) - exit(1) + sys.exit(1) diff --git a/.ci/lib/verify_checksums.py b/.ci/lib/verify_checksums.py index f9cb0bac5..f8864f944 100755 --- a/.ci/lib/verify_checksums.py +++ b/.ci/lib/verify_checksums.py @@ -19,8 +19,9 @@ def verify_checksums(packages): print("verifying checksums: " + ", ".join(packages)) common.run_pmbootstrap(["build_init"]) - common.run_pmbootstrap(["--details-to-stdout", "checksum", "--verify"] + - list(packages)) + common.run_pmbootstrap( + ["--details-to-stdout", "checksum", "--verify"] + list(packages) + ) if __name__ == "__main__": @@ -39,14 +40,14 @@ if __name__ == "__main__": archived = set() for path in pmb.core.pkgrepo.pkgrepo_iter_package_dirs(): # Store archived packages for matching later - if os.path.split(os.path.split(path)[0])[1] == 'archived': + if os.path.split(os.path.split(path)[0])[1] == "archived": archived.add(os.path.basename(path)) continue buildable_pkgs.add(os.path.basename(path)) # To store a list of packages from extra-repos/systemd for special handling # later: - systemd_pkgs = list() + systemd_pkgs = [] # Filter out packages are not found in enabled repos or are in archived # (Iterate over copy of packages, because we modify it in this loop) @@ -73,7 +74,7 @@ if __name__ == "__main__": if len(packages) == 0: print("no packages to check for the current repo") else: - verify_checksums(packages) + verify_checksums(packages) # FIXME: this should probably be more generic, if other repos are added later? if systemd_pkgs: diff --git a/.ci/ruff.sh b/.ci/ruff.sh index be9a4f34a..aff6eb034 100755 --- a/.ci/ruff.sh +++ b/.ci/ruff.sh @@ -11,4 +11,4 @@ fi set -x # shellcheck disable=SC2046 -ruff check $(find .ci -name '*.py') +ruff check --ignore LOG015 $(find .ci -name '*.py') diff --git a/.ci/testcases/test_device.py b/.ci/testcases/test_device.py index 0559b51dc..05cb3dd6a 100644 --- a/.ci/testcases/test_device.py +++ b/.ci/testcases/test_device.py @@ -15,13 +15,15 @@ gitlab_groups = [ def device_dependency_check(apkbuild, path): - """ Raise an error if a device package has a dependency that is not allowed - (e.g. because it should be in a subpackage instead). """ + """Raise an error if a device package has a dependency that is not allowed + (e.g. because it should be in a subpackage instead).""" for depend in apkbuild["depends"]: if depend == "mesa-dri-gallium": - raise RuntimeError(f"{path}: mesa-dri-gallium shouldn't be in" - " depends anymore (see pmaports!3478)") + raise RuntimeError( + f"{path}: mesa-dri-gallium shouldn't be in" + " depends anymore (see pmaports!3478)" + ) def test_aports_device(): @@ -44,11 +46,12 @@ def test_aports_device(): device_dependency_check(apkbuild, path) # Architecture - device = apkbuild["pkgname"][len("device-"):] + device = apkbuild["pkgname"][len("device-") :] deviceinfo = pmb.parse.deviceinfo(device) if Arch.from_str("".join(apkbuild["arch"])) != deviceinfo.arch: - raise RuntimeError("wrong architecture, please change to arch=\"" - f"{deviceinfo.arch}\": {path}") + raise RuntimeError( + f'wrong architecture, please change to arch="{deviceinfo.arch}": {path}' + ) if "!archcheck" not in apkbuild["options"]: raise RuntimeError(f"!archcheck missing in options= line: {path}") @@ -64,7 +67,7 @@ def test_aports_device_kernel(): for path in aports.glob("device/*/device-*/APKBUILD"): # Parse apkbuild and kernels from subpackages apkbuild = pmb.parse.apkbuild(path) - device = apkbuild["pkgname"][len("device-"):] + device = apkbuild["pkgname"][len("device-") :] kernels_subpackages = pmb.parse._apkbuild.kernels(device) # Parse kernels from depends @@ -76,20 +79,24 @@ def test_aports_device_kernel(): # Kernel in subpackages *and* depends if kernels_subpackages: - raise RuntimeError(f"Kernel package '{depend}' needs to" - " be removed when using kernel" + - f" subpackages: {path}") + raise RuntimeError( + f"Kernel package '{depend}' needs to" + " be removed when using kernel" + f" subpackages: {path}" + ) # No kernel if not kernels_depends and not kernels_subpackages: - raise RuntimeError("Device doesn't have a kernel in depends or" - f" subpackages: {path}") + raise RuntimeError( + f"Device doesn't have a kernel in depends or subpackages: {path}" + ) # Multiple kernels in depends if len(kernels_depends) > 1: - raise RuntimeError("Please use kernel subpackages instead of" - " multiple kernels in depends (see" - f" ): {path}") + raise RuntimeError( + "Please use kernel subpackages instead of" + " multiple kernels in depends (see" + f" ): {path}" + ) # @pytest.mark.xfail # Not all aports have been updated yet @@ -100,14 +107,15 @@ def test_aports_maintained(): """ for path in pkgrepo_iglob("device/main/*/APKBUILD"): - if 'firmware-' in path.parent.name: + if "firmware-" in path.parent.name: continue maintainers = pmb.parse._apkbuild.maintainers(path) - assert maintainers and len(maintainers) >= 2, \ + assert maintainers and len(maintainers) >= 2, ( f"{path} in main needs at least 1 Maintainer and 1 Co-Maintainer" + ) for path in pkgrepo_iglob("device/community/*/APKBUILD"): - if 'firmware-' in path.parent.name: + if "firmware-" in path.parent.name: continue maintainers = pmb.parse._apkbuild.maintainers(path) assert maintainers, f"{path} in community needs at least 1 Maintainer" @@ -121,5 +129,7 @@ def test_aports_archived(): for path in pkgrepo_iglob("device/archived/*/APKBUILD"): archived = pmb.parse._apkbuild.archived(path) - assert archived, f"{path} should have an Archived: " +\ - "comment that describes why the package is archived" + assert archived, ( + f"{path} should have an Archived: " + + "comment that describes why the package is archived" + ) diff --git a/.ci/testcases/test_deviceinfo.py b/.ci/testcases/test_deviceinfo.py index b2976cf91..b653823a0 100644 --- a/.ci/testcases/test_deviceinfo.py +++ b/.ci/testcases/test_deviceinfo.py @@ -14,14 +14,13 @@ def test_deviceinfo(): # Iterate over all devices last_exception = None count = 0 - pattern = re.compile("^deviceinfo_[a-zA-Z0-9_]*=\".*\"(\\s*# .*)?$") + pattern = re.compile('^deviceinfo_[a-zA-Z0-9_]*=".*"(\\s*# .*)?$') for folder in pkgrepo_iglob("device/*/device-*"): device = folder.name.split("-", 1)[1] - f = open(folder / "deviceinfo") - lines = f.read().split("\n") - f.close() + with open(folder / "deviceinfo") as f: + lines = f.read().split("\n") try: for line in lines: @@ -30,10 +29,12 @@ def test_deviceinfo(): continue # Check line against regex (can't use multiple lines etc.) - if not pattern.match(line) or line.endswith("\\\""): - raise RuntimeError("Line looks invalid, maybe missing" - " quotes/multi-line string/malformed" - f" inline comment? {line}") + if not pattern.match(line) or line.endswith('\\"'): + raise RuntimeError( + "Line looks invalid, maybe missing" + " quotes/multi-line string/malformed" + f" inline comment? {line}" + ) # Successful deviceinfo parsing / obsolete options info = pmb.parse.deviceinfo(device) @@ -41,15 +42,18 @@ def test_deviceinfo(): # deviceinfo_name must start with manufacturer name = info.name manufacturer = info.manufacturer - if not name.startswith(manufacturer) and \ - not name.startswith("Google") and \ - not name.startswith("Jolla"): - raise RuntimeError("Please add the manufacturer in front of" - " the deviceinfo_name, e.g.: '" + - manufacturer + " " + name + "'") + if ( + not name.startswith(manufacturer) + and not name.startswith("Google") + and not name.startswith("Jolla") + ): + raise RuntimeError( + "Please add the manufacturer in front of" + " the deviceinfo_name, e.g.: '" + manufacturer + " " + name + "'" + ) # Don't abort on first error - except Exception as e: + except (RuntimeError, SyntaxError, ValueError) as e: last_exception = e count += 1 print(device + ": " + str(e)) diff --git a/.ci/testcases/test_directory_structure.py b/.ci/testcases/test_directory_structure.py index 2d77fe986..2881f7f88 100644 --- a/.ci/testcases/test_directory_structure.py +++ b/.ci/testcases/test_directory_structure.py @@ -23,9 +23,11 @@ expected_directories = [ # This test makes sure all of them are in one of the expected locations. def test_directories(): apkbuilds = set(glob.iglob("**/APKBUILD", recursive=True)) - expected = set(f for d in expected_directories for f in glob.iglob(d + "/*/APKBUILD")) - assert apkbuilds == expected, "Found APKBUILD in unexpected directory. " \ + expected = {f for d in expected_directories for f in glob.iglob(d + "/*/APKBUILD")} + assert apkbuilds == expected, ( + "Found APKBUILD in unexpected directory. " "Note that we moved firmware/* to device/{main,community,testing}/*." + ) # Ensure no file in pmaports are executable. @@ -38,11 +40,13 @@ def test_executable_files(): permissions = os.stat(file).st_mode executable_bits = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH if permissions & executable_bits != 0: - raise RuntimeError(f"\"{file}\" is executable. Files in pmaports" + - " should not be executables. post-* files" + - " don't need to be executable and executables" + - " should be installed using `install -D" + - "m0755` or a variation thereof.") + raise RuntimeError( + f'"{file}" is executable. Files in pmaports' + + " should not be executables. post-* files" + + " don't need to be executable and executables" + + " should be installed using `install -D" + + "m0755` or a variation thereof." + ) # Make sure files belong to a package (below a directory with APKBUILD) @@ -54,24 +58,26 @@ def test_files_belong_to_package(): # Skip "hidden" directories (.ci/, device/.shared-patches/) dirs[:] = [d for d in dirs if not d.startswith(".")] # Ignore files in root directory (README.md) - if dirpath == '.': + if dirpath == ".": continue # Ignore files under docs/ - if dirpath.startswith('./docs'): + if dirpath.startswith("./docs"): continue # Switched to another directory? if package_dir and not dirpath.startswith(package_dir + os.sep): package_dir = None - if 'APKBUILD' in files: - assert not package_dir, f"Nested packages: {package_dir} and {dirpath} " \ - "both contain an APKBUILD" + if "APKBUILD" in files: + assert not package_dir, ( + f"Nested packages: {package_dir} and {dirpath} both contain an APKBUILD" + ) package_dir = dirpath # Skip specific ".gitlab-ci.yml" files outside of usual structure if dirpath in ["./cross"]: files[:] = [f for f in files if f != ".gitlab-ci.yml"] - assert not files or package_dir, "Found files that do not belong to any package: " \ - f"{dirpath}/{files}" + assert not files or package_dir, ( + f"Found files that do not belong to any package: {dirpath}/{files}" + ) diff --git a/.ci/testcases/test_firmware.py b/.ci/testcases/test_firmware.py index d12540e6d..e9c3b46ed 100644 --- a/.ci/testcases/test_firmware.py +++ b/.ci/testcases/test_firmware.py @@ -22,28 +22,33 @@ def test_aports_firmware(): "firmware-samsung-maguro", # Depends on firmware-aosp-broadcom-wlan "firmware-xiaomi-ferrari", # Depends on soc-qcom-msm8916 "firmware-xiaomi-willow", # Doesn't build, source link is dead (pma#1212) - "firmware-samsung-i9105p", # Depends on firmware-aosp-broadcom-wlan + "firmware-samsung-i9105p", # Depends on firmware-aosp-broadcom-wlan ] for path in pkgrepo_iglob("**/firmware-*/APKBUILD", recursive=True): apkbuild = pmb.parse.apkbuild(path) aport_name = os.path.basename(path.parent) - if aport_name not in excluded: - if "pmb:cross-native" not in apkbuild["options"]: - raise RuntimeError(f"{aport_name}: \"pmb:cross-native\" missing in" - " options= line. The pmb:cross-native option is" - " preferred because it results in significantly" - " lower build times. If the package doesn't build" - " with the option, you can add an exemption in" - " .gitlab-ci/testcases/test_firmware.py.") + if aport_name not in excluded and "pmb:cross-native" not in apkbuild["options"]: + raise RuntimeError( + f'{aport_name}: "pmb:cross-native" missing in' + " options= line. The pmb:cross-native option is" + " preferred because it results in significantly" + " lower build times. If the package doesn't build" + " with the option, you can add an exemption in" + " .gitlab-ci/testcases/test_firmware.py." + ) if "!tracedeps" not in apkbuild["options"]: - raise RuntimeError(f"{aport_name}: \"!tracedeps\" missing in" - " options= line. The tracedeps option is superfluous" - " for firmware packages.") + raise RuntimeError( + f'{aport_name}: "!tracedeps" missing in' + " options= line. The tracedeps option is superfluous" + " for firmware packages." + ) if "noarch" in apkbuild["arch"]: - raise RuntimeError(f"{aport_name}: \"arch\" must not be \"noarch\"!" - " Please limit this firmware package to the" - " required architectures only!") + raise RuntimeError( + f'{aport_name}: "arch" must not be "noarch"!' + " Please limit this firmware package to the" + " required architectures only!" + ) diff --git a/.ci/testcases/test_framework_versions.py b/.ci/testcases/test_framework_versions.py index 534c617b9..9cae738dc 100644 --- a/.ci/testcases/test_framework_versions.py +++ b/.ci/testcases/test_framework_versions.py @@ -1,9 +1,9 @@ # Copyright 2021 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later -""" Make sure that components of frameworks (Qt, KDE, ...) have the same - version in all packages. We scan all packages, categorize them by URL - and then compare the versions of all packages in each category. """ +"""Make sure that components of frameworks (Qt, KDE, ...) have the same +version in all packages. We scan all packages, categorize them by URL +and then compare the versions of all packages in each category.""" import logging @@ -41,8 +41,7 @@ def get_categorized_packages(): category = "plasma" elif "https://community.kde.org/Frameworks" in url: category = "kde" - elif url in ["http://qt-project.org/", - "https://www.qt.io/developers/"]: + elif url in ["http://qt-project.org/", "https://www.qt.io/developers/"]: category = "qt" # Remove hotfix number (i.e. 5.16.90.1 becomes 5.16.90) @@ -69,7 +68,6 @@ def check_categories(categories): for category, packages in categories.items(): reference = None for pkgname, pkgver in packages.items(): - # Use the first package as reference and print a summary if not reference: logging.info("---") @@ -87,8 +85,9 @@ def check_categories(categories): # Print the reference and skip checking it against itself reference = {"pkgname": pkgname, "pkgver": pkgver} - logging.info("Reference pkgver: " + pkgver + " (from '" + - pkgname + "')") + logging.info( + "Reference pkgver: " + pkgver + " (from '" + pkgname + "')" + ) continue # Check version against reference @@ -98,8 +97,7 @@ def check_categories(categories): # Each category must at least have one package if not reference: - logging.info("ERROR: could not find any packages in category: " + - category) + logging.info("ERROR: could not find any packages in category: " + category) ret = False return ret diff --git a/.ci/testcases/test_kernel.py b/.ci/testcases/test_kernel.py index 425a12607..da38d7609 100644 --- a/.ci/testcases/test_kernel.py +++ b/.ci/testcases/test_kernel.py @@ -21,29 +21,48 @@ def test_aports_kernel(): continue # This package isn't a linux kernel! if "pmb:cross-native" not in apkbuild["options"]: - raise RuntimeError(f"{aport_name}: \"pmb:cross-native\" missing in" - " options= line") + raise RuntimeError( + f'{aport_name}: "pmb:cross-native" missing in options= line' + ) # cross-compilers should not be in makedepends - for ccc in ["gcc-armv7", "gcc-armhf", "gcc-aarch64", - "gcc4-armv7", "gcc4-armhf", "gcc4-aarch64", - "gcc6-armv7", "gcc6-armhf", "gcc6-aarch64"]: + for ccc in [ + "gcc-armv7", + "gcc-armhf", + "gcc-aarch64", + "gcc4-armv7", + "gcc4-armhf", + "gcc4-aarch64", + "gcc6-armv7", + "gcc6-armhf", + "gcc6-aarch64", + ]: if ccc in apkbuild["makedepends"]: - raise RuntimeError(f"{aport_name}: Cross-compiler ({ccc}) should" - " not be explicitly specified in makedepends!" - " pmbootstrap installs cross-compiler" - " automatically.") + raise RuntimeError( + f"{aport_name}: Cross-compiler ({ccc}) should" + " not be explicitly specified in makedepends!" + " pmbootstrap installs cross-compiler" + " automatically." + ) # check some options only for main and community devices _, relpath = pkgrepo_relative_path(path) for part in ["main", "community"]: - if part in relpath.parts: - if "pmb:kconfigcheck-community" not in apkbuild["options"]: - raise RuntimeError(f"{aport_name}: \"pmb:kconfigcheck-community\" missing in" - " options= line, required for all community/main devices.") + if ( + part in relpath.parts + and "pmb:kconfigcheck-community" not in apkbuild["options"] + ): + raise RuntimeError( + f'{aport_name}: "pmb:kconfigcheck-community" missing in' + " options= line, required for all community/main devices." + ) # check for postmarketos-installkernel in makedepends when installing kernel with make - if bool(re.search("make z?install", path.read_text(encoding="utf-8"))): - if "postmarketos-installkernel" not in apkbuild["makedepends"]: - raise RuntimeError(f"{aport_name}: \"postmarketos-installkernel\" missing in" - " makedepends, required when using make install/zinstall.") + if ( + bool(re.search("make z?install", path.read_text(encoding="utf-8"))) + and "postmarketos-installkernel" not in apkbuild["makedepends"] + ): + raise RuntimeError( + f'{aport_name}: "postmarketos-installkernel" missing in' + " makedepends, required when using make install/zinstall." + ) diff --git a/.ci/testcases/test_provides.py b/.ci/testcases/test_provides.py index bde58bddf..ab759a866 100644 --- a/.ci/testcases/test_provides.py +++ b/.ci/testcases/test_provides.py @@ -44,7 +44,9 @@ def apkbuild_check_provides(path, apkbuild, version, pkgname, subpkgname=None): if not provide.endswith(f"={version}"): # Valid version strings, per the APKBUILD reference, are fine. This regex # attempts to detect those. - pattern = r'^\d+(\.\d+)*[a-z]?(_(?:alpha|beta|pre|rc|cvs|svn|git|hg|p)\d*)*-r\d+$' + pattern = ( + r"^\d+(\.\d+)*[a-z]?(_(?:alpha|beta|pre|rc|cvs|svn|git|hg|p)\d*)*-r\d+$" + ) [provide_no_ver, provide_ver] = provide.split("=", 1) if not re.match(pattern, provide_ver): error = f"error in , provide_ver{path}:\n" @@ -62,7 +64,9 @@ def apkbuild_check_provides(path, apkbuild, version, pkgname, subpkgname=None): error += " version on purpose, you also need to set a" error += " provider_priority (pma#1766).\n" error += "Reference:" - error += " https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#provides" + error += ( + " https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#provides" + ) error += f"\nVERSION: {provide_no_ver}" ret += [error] return ret @@ -82,8 +86,9 @@ def test_provides(): if not subpkg_data: # default packaging function like -doc continue - errors += apkbuild_check_provides(path_rel, subpkg_data, version, - pkgname, subpkg) + errors += apkbuild_check_provides( + path_rel, subpkg_data, version, pkgname, subpkg + ) if errors: for error in errors: diff --git a/.ci/testcases/test_source.py b/.ci/testcases/test_source.py index efa076471..c098b3881 100644 --- a/.ci/testcases/test_source.py +++ b/.ci/testcases/test_source.py @@ -45,9 +45,11 @@ def parse_source_from_checksums(apkbuild_path): try: checksum, filename = line.rstrip().split(" ", 2) except ValueError: - raise ValueError("Failed to parse checksums. Try to delete the" - " checksums and generate them again with" - f" 'pmbootstrap checksum': {apkbuild_path}") + raise ValueError( + "Failed to parse checksums. Try to delete the" + " checksums and generate them again with" + f" 'pmbootstrap checksum': {apkbuild_path}" + ) # Cut off 'sha512sums="' if the first checksum is in that line checksum = checksum.removeprefix(start) @@ -96,15 +98,21 @@ def test_aports_unreferenced_files(): if rel_file_path in ["APKBUILD", "gitlab-ci.yml.j2"] or os.path.isdir(file): continue # Skip kernel fragments in linux-* packages - if os.path.basename(rel_file_path).endswith(".config") and apkbuild["pkgname"].startswith("linux-"): + if os.path.basename(rel_file_path).endswith(".config") and apkbuild[ + "pkgname" + ].startswith("linux-"): continue - if os.path.basename(rel_file_path) not in sources_chk \ - and rel_file_path not in apkbuild["install"] \ - and rel_file_path not in subpackage_installs \ - and rel_file_path not in trigger_sources \ - and not rel_file_path.startswith("tests/"): - raise RuntimeError(f"{apkbuild_path}: found unreferenced file: {rel_file_path}") + if ( + os.path.basename(rel_file_path) not in sources_chk + and rel_file_path not in apkbuild["install"] + and rel_file_path not in subpackage_installs + and rel_file_path not in trigger_sources + and not rel_file_path.startswith("tests/") + ): + raise RuntimeError( + f"{apkbuild_path}: found unreferenced file: {rel_file_path}" + ) def test_distfiles_conflict(): @@ -131,8 +139,10 @@ def test_distfiles_conflict(): # First time seeing this file if filename not in source_all: - source_all[filename] = {"checksum": checksum, - "apkbuild_rel": apkbuild_rel} + source_all[filename] = { + "checksum": checksum, + "apkbuild_rel": apkbuild_rel, + } continue # Saw this file already with same checksum @@ -141,15 +151,19 @@ def test_distfiles_conflict(): # Saw this file already with different checksum logging.error("") - logging.error(f"ERROR: the source file '{filename}' has different" - " checksums in the following files:") + logging.error( + f"ERROR: the source file '{filename}' has different" + " checksums in the following files:" + ) logging.error(f"- {source_all[filename]['apkbuild_rel']}:") logging.error(f" {source_all[filename]['checksum']}") logging.error(f"- {apkbuild_rel}:") logging.error(f" {checksum}") logging.error("") - logging.error("Fix this by setting a different target filename in" - " the package you modified:") + logging.error( + "Fix this by setting a different target filename in" + " the package you modified:" + ) logging.error("https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#source") logging.error("") raise RuntimeError(f"Conflict with source file '{filename}'") diff --git a/.ci/testcases/test_ui.py b/.ci/testcases/test_ui.py index f1149903c..32041d223 100644 --- a/.ci/testcases/test_ui.py +++ b/.ci/testcases/test_ui.py @@ -31,8 +31,10 @@ def test_aports_ui(): for package in apkbuild["_pmb_recommends"]: depend = pmb.helpers.package.get(package, arch, must_exist=False) if depend is None: - raise RuntimeError(f"{path}: package '{package}' from" - f" _pmb_recommends not found for arch '{arch}'") + raise RuntimeError( + f"{path}: package '{package}' from" + f" _pmb_recommends not found for arch '{arch}'" + ) # Check packages from "_pmb_recommends" of -extras subpackage if one exists if f"{apkbuild['pkgname']}-extras" in apkbuild["subpackages"]: @@ -40,5 +42,7 @@ def test_aports_ui(): for package in apkbuild["_pmb_recommends"]: depend = pmb.helpers.package.get(package, arch, must_exist=False) if depend is None: - raise RuntimeError(f"{path}: package '{package}' from _pmb_recommends " - f"of -extras subpackage is not found for arch '{arch}'") + raise RuntimeError( + f"{path}: package '{package}' from _pmb_recommends " + f"of -extras subpackage is not found for arch '{arch}'" + )