snlavaise/pmaports-rtwo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci: fix remaining ruff issues and run ruff format

Browse Source 
We only disable LOG015 rule, as in the simple situation of pmaports,
it doesn't matter too much if in pytest we are using the root logger
or a custom one.

Signed-off-by: Pablo Correa Gómez <pabloyoyoista@postmarketos.org>
Part-of: <https://gitlab.postmarketos.org/postmarketOS/pmaports/-/merge_requests/9363>
This commit is contained in:
Pablo Correa Gómez
2026-08-12 05:39:47 +00:00
committed by The Friendly Meow (merge) Bot
parent 6c7b46ef97
commit d035338047
19 changed files with 374 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/lib/apkbuild_linting.py
+12 -6
View File
@@ -21,7 +21,7 @@ custom_valid_options = [
]
def get_kconfigcheck_categories() -> list [str]:
def get_kconfigcheck_categories() -> list[str]:
with open("kconfigcheck.toml", "rb") as kconfigcheck_config:
kconfigcheck_data = tomllib.load(kconfigcheck_config)
@@ -34,27 +34,33 @@ def get_kconfigcheck_categories() -> list [str]:
else:
kconfigcheck_categories.append(category_name)
return [item.replace("category:", "pmb:kconfigcheck-") for item in kconfigcheck_categories]
return [
item.replace("category:", "pmb:kconfigcheck-")
for item in kconfigcheck_categories
]
if __name__ == "__main__":
if common.commit_message_has_string("[ci:skip-apkbuild-lint]"):
print("WARNING: not linting deviceinfo files ([ci:skip-apkbuild-lint])")
exit(0)
sys.exit(0)
kconfigcheck_categories = get_kconfigcheck_categories()
custom_valid_options += kconfigcheck_categories
os.environ["CUSTOM_VALID_OPTIONS"] = " ".join(custom_valid_options)
apkbuilds = {file for file in common.get_changed_files()
if os.path.basename(file) == "APKBUILD"}
apkbuilds = {
file
for file in common.get_changed_files()
if os.path.basename(file) == "APKBUILD"
}
if len(apkbuilds) < 1:
print("No APKBUILDs to lint")
sys.exit(0)
apkbuilds_filtered = []
for apkbuild in apkbuilds:
if apkbuild.startswith("temp/") or apkbuild.startswith("cross/"):
if apkbuild.startswith(("temp/", "cross/")):
print(f"NOTE: Skipping linting of {apkbuild}")
continue
apkbuilds_filtered.append(apkbuild)
.ci/lib/build_changed_aports.py
+24 -9
View File
@@ -21,11 +21,20 @@ def build_strict(packages, arch: Arch):
common.run_pmbootstrap(["build_init"])
# We set the timeout to 1 hour because linking the Linux kernel with
# ThinLTO on our aarch64 runners takes slightly over 30 minutes
common.run_pmbootstrap(["--details-to-stdout", "--no-ccache",
"--timeout", "3600",
"build",
"--strict", "--force",
"--arch", str(arch), ] + list(packages))
common.run_pmbootstrap(
[
"--details-to-stdout",
"--no-ccache",
"--timeout",
"3600",
"build",
"--strict",
"--force",
"--arch",
str(arch),
]
+ list(packages)
)
if __name__ == "__main__":
@@ -51,7 +60,7 @@ if __name__ == "__main__":
buildable_pkgs.add(os.path.basename(path))
# To store a list of packages from extra-repos/systemd for special handling later:
systemd_pkgs = list()
systemd_pkgs = []
# Filter out packages that either:
# 1. can't be built for given arch
@@ -92,10 +101,16 @@ if __name__ == "__main__":
# filter out packages that can't be built for arch
# (Iterate over copy of `systemd_pkgs`, because we modify it in this loop)
for package in systemd_pkgs.copy():
apkbuild_path = pmb.helpers.pmaports.find(package, True, True, with_extra_repos="enabled")
apkbuild = pmb.parse._apkbuild.apkbuild(pathlib.Path(apkbuild_path, "APKBUILD"))
apkbuild_path = pmb.helpers.pmaports.find(
package, True, True, with_extra_repos="enabled"
)
apkbuild = pmb.parse._apkbuild.apkbuild(
pathlib.Path(apkbuild_path, "APKBUILD")
)
if arch not in Arch.from_arch_field(apkbuild["arch"]):
print(f"(extra-repos/systemd) {package}: not enabled for {arch}, skipping")
print(
f"(extra-repos/systemd) {package}: not enabled for {arch}, skipping"
)
systemd_pkgs.remove(package)
# No packages: skip build
.ci/lib/check_changed_versions.py
+39 -27
View File
@@ -26,8 +26,7 @@ def get_package_contents(package, revision, check=True):
stderr = subprocess.DEVNULL
# Run something like "git show upstream/main:main/hello-world/APKBUILD"
apkbuild_content = common.run_git(["show", revision + ":" + package], check,
stderr)
apkbuild_content = common.run_git(["show", revision + ":" + package], check, stderr)
if not apkbuild_content:
return None
@@ -36,15 +35,14 @@ def get_package_contents(package, revision, check=True):
with tempfile.TemporaryDirectory() as tempdir:
with open(tempdir + "/APKBUILD", "w", encoding="utf-8") as handle:
handle.write(apkbuild_content)
parsed = pmb.parse.apkbuild(pathlib.Path(tempdir, "APKBUILD"),
False, False)
parsed = pmb.parse.apkbuild(pathlib.Path(tempdir, "APKBUILD"), False, False)
return parsed
def get_package_version(package, revision, check=True):
""" returns version in the format "{pkgver}-r{pkgrel}", or None if no
matching package is found """
"""returns version in the format "{pkgver}-r{pkgrel}", or None if no
matching package is found"""
parsed = get_package_contents(package, revision, check)
if parsed is None:
return None
@@ -52,7 +50,7 @@ def get_package_version(package, revision, check=True):
def version_compare_operator(result):
""" :param result: return value from pmb.parse.version.compare() """
""":param result: return value from pmb.parse.version.compare()"""
if result == -1:
return "<"
elif result == 0:
@@ -88,7 +86,7 @@ def exit_with_error_message():
print(" latest commit message, then force push.")
print()
print("Thank you and sorry for the inconvenience.")
exit(1)
sys.exit(1)
def check_versions(packages):
@@ -101,14 +99,21 @@ def check_versions(packages):
# desired here, since we already know what packages changed, and really
# want to check if the version was increased towards *current* upstream
# branch HEAD.
common.run_git(["remote", "add", "upstream",
"https://gitlab.postmarketos.org/postmarketOS/pmaports.git"], False)
common.run_git(
[
"remote",
"add",
"upstream",
"https://gitlab.postmarketos.org/postmarketOS/pmaports.git",
],
False,
)
common.run_git(["fetch", "-q", "upstream"])
commit = f"upstream/{common.get_upstream_branch()}"
if common.run_git(["rev-parse", "HEAD"]) == common.run_git(["rev-parse",
commit]):
print(f"NOTE: {commit} is on same commit as HEAD, comparing"
" HEAD against HEAD~1.")
if common.run_git(["rev-parse", "HEAD"]) == common.run_git(["rev-parse", commit]):
print(
f"NOTE: {commit} is on same commit as HEAD, comparing HEAD against HEAD~1."
)
commit = "HEAD~1"
# Exclude linux-next from pkgrel checks, as it is updated automatically.
@@ -124,28 +129,35 @@ def check_versions(packages):
head = get_package_version(package, "HEAD")
upstream = get_package_version(package, commit, False)
if not upstream:
if head.rpartition('r')[2] != "0":
if head.rpartition("r")[2] != "0":
print(f"- {package}: {head} (HEAD) (new package) [ERROR]")
error = True
else:
print(f"- {package}: {head} (HEAD) (new package)")
continue
is_device_pkg = os.path.basename(package).startswith('device-')
is_device_pkg = os.path.basename(package).startswith("device-")
if is_device_pkg:
# Check pkgver follows expected format for device packages
pkgver = head.rpartition('-r')[0]
pkgver = head.rpartition("-r")[0]
if not pkgver.isdigit():
print(f" - {package}: invalid pkgver \"{pkgver}\""
"See: https://docs.postmarketos.org/pmaports/main/packaging-guidelines.html#package-versioning-pkgver-pkgrel")
print(
f' - {package}: invalid pkgver "{pkgver}"'
"See: https://docs.postmarketos.org/pmaports/main/packaging-guidelines.html#package-versioning-pkgver-pkgrel"
)
error = True
# Check that pkgrel was reset to 0 when pkgver was changed
head_parsed = get_package_contents(package, "HEAD", False)
upstream_parsed = get_package_contents(package, commit, False)
if head_parsed["pkgver"] != upstream_parsed["pkgver"] and head_parsed["pkgrel"] != "0":
print(f" - {package}: pkgrel should be 0 when pkgver changes."
"See: https://docs.postmarketos.org/pmaports/main/packaging-guidelines.html#package-versioning-pkgver-pkgrel")
if (
head_parsed["pkgver"] != upstream_parsed["pkgver"]
and head_parsed["pkgrel"] != "0"
):
print(
f" - {package}: pkgrel should be 0 when pkgver changes."
"See: https://docs.postmarketos.org/pmaports/main/packaging-guidelines.html#package-versioning-pkgver-pkgrel"
)
error = True
# Compare head and upstream versions
@@ -169,15 +181,16 @@ if __name__ == "__main__":
packages = common.get_changed_packages()
if len(packages) == 0:
print("no aports changed in this branch")
exit(0)
sys.exit(0)
print(f"Changed packages: {packages}")
# Potentially skip this check
if common.commit_message_has_string("[ci:skip-vercheck]"):
print("WARNING: not checking for changed package versions"
" ([ci:skip-vercheck])!")
exit(0)
print(
"WARNING: not checking for changed package versions ([ci:skip-vercheck])!"
)
sys.exit(0)
# Initialize args (so we can use pmbootstrap's APKBUILD parsing)
sys.argv = ["pmbootstrap.py", "chroot"]
@@ -187,7 +200,6 @@ if __name__ == "__main__":
# pmb.helpers.logging.init(args)
pmb.helpers.logging.init(context.log, args.verbose, context.details_to_stdout)
# Verify package versions
print("checking changed package versions...")
check_versions(packages)
.ci/lib/check_devices_in_wiki.py
+35 -23
View File
@@ -22,7 +22,7 @@ def get_devices() -> list[str]:
# devices that have a working mainline kernel. Those are usually
# unmaintained and therefore might not appear in the wiki. However,
# the main device should be documented (remove the -downstream suffix).
device = device.removesuffix('-downstream')
device = device.removesuffix("-downstream")
ret.append(device)
return sorted(ret)
@@ -30,10 +30,10 @@ def get_devices() -> list[str]:
def get_wiki_devices_html(path: Path | None) -> dict[str, str]:
""":param path: to a local file with the saved content of the devices wiki
page or None to download a fresh copy
:returns: HTML of the page, split into booting and not booting:
{"booting": "<!DOCTYPE HTML>\n<html..."
"not_booting": "Not booting</span></h2>\n<p>These..."}"""
page or None to download a fresh copy
:returns: HTML of the page, split into booting and not booting:
{"booting": "<!DOCTYPE HTML>\n<html..."
"not_booting": "Not booting</span></h2>\n<p>These..."}"""
content = ""
if path:
# Read file
@@ -46,7 +46,7 @@ def get_wiki_devices_html(path: Path | None) -> dict[str, str]:
content = f.read().decode("utf-8")
# Split into booting and not booting
split = content.split("<span class=\"mw-headline\" id=\"Non-booting_devices\">")
split = content.split('<span class="mw-headline" id="Non-booting_devices">')
if len(split) != 2:
print("*** Failed to parse wiki page")
@@ -65,16 +65,18 @@ def get_wiki_renamed_devices_html() -> str:
def check_device(device: str, html: dict[str, str], is_booting: bool) -> bool:
""":param is_booting: require the device to be in the booting section, not
just anywhere in the page (i.e. in the not booting
table).
:returns: True when the device is in the appropriate section."""
just anywhere in the page (i.e. in the not booting
table).
:returns: True when the device is in the appropriate section."""
if device in html["booting"]:
return True
if device in html["not_booting"]:
if is_booting:
print(device + ": still in 'not booting' section (if this is a"
" merge request, your device should be in the booting"
" section already)")
print(
device + ": still in 'not booting' section (if this is a"
" merge request, your device should be in the booting"
" section already)"
)
return False
return True
if device in html["renamed"]:
@@ -88,17 +90,27 @@ def check_device(device: str, html: dict[str, str], is_booting: bool) -> bool:
def main() -> int:
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument("--booting", help="devices must be in the upper table,"
" being in the 'not booting' table below is not"
" enough (all devices in pmaports main and stable"
" branches and should be in the upper table)",
action="store_true")
parser.add_argument("--path", help="instead of downloading the devices"
" page from the wiki, use a local HTML file",
type=Path, default=None)
parser.add_argument("--all", help="check all devices (default: check only"
" devices changed from branch point)",
action="store_true")
parser.add_argument(
"--booting",
help="devices must be in the upper table,"
" being in the 'not booting' table below is not"
" enough (all devices in pmaports main and stable"
" branches and should be in the upper table)",
action="store_true",
)
parser.add_argument(
"--path",
help="instead of downloading the devices"
" page from the wiki, use a local HTML file",
type=Path,
default=None,
)
parser.add_argument(
"--all",
help="check all devices (default: check only"
" devices changed from branch point)",
action="store_true",
)
args = parser.parse_args()
html = get_wiki_devices_html(args.path)
.ci/lib/common.py
+35 -21
View File
@@ -16,7 +16,7 @@ def get_pmaports_dir():
def run_git(parameters, check=True, stderr=None):
""" Run git in the pmaports dir and return the output """
"""Run git in the pmaports dir and return the output"""
cmd = ["git", "-C", get_pmaports_dir()] + parameters
try:
return subprocess.check_output(cmd, stderr=stderr).decode()
@@ -34,23 +34,25 @@ def commit_message_has_string(needle):
def all_committed_by_merge_bot():
base_commit = get_base_commit()
commit_emails = run_git(["log", "--pretty=format:%ce", f"{base_commit}..HEAD"]).splitlines()
commit_emails = run_git(
["log", "--pretty=format:%ce", f"{base_commit}..HEAD"]
).splitlines()
bot_email = os.getenv("PMOS_MERGE_BOT_EMAIL", "merge-bot@postmarketos.org")
return all(bot_email == ce for ce in commit_emails)
def run_pmbootstrap(parameters):
""" Run pmbootstrap with the pmaports dir as --aports """
"""Run pmbootstrap with the pmaports dir as --aports"""
cmd = ["pmbootstrap", "--aports", get_pmaports_dir()] + parameters
subprocess.run(cmd, text=True, check=True)
@cache
def get_upstream_branch():
""" Use pmaports.cfg from current branch (e.g. "v20.05_fix-ci") and
channels.cfg from main to retrieve the upstream branch.
"""Use pmaports.cfg from current branch (e.g. "v20.05_fix-ci") and
channels.cfg from main to retrieve the upstream branch.
:returns: branch name, e.g. "v20.05" """
:returns: branch name, e.g. "v20.05" """
# Prefer gitlab CI target branch name if it's set (i.e. running in gitlab CI)
if target_branch := os.environ.get("CI_MERGE_REQUEST_TARGET_BRANCH_NAME"):
@@ -68,10 +70,11 @@ def get_upstream_branch():
channels_cfg_str = run_git(["show", "upstream/main:channels.cfg"])
channels_cfg = configparser.ConfigParser()
channels_cfg.read_string(channels_cfg_str)
assert channel in channels_cfg, \
f"Channel '{channel}' from pmaports.cfg in your branch is unknown." \
" This appears to be an old branch, consider recreating your change" \
assert channel in channels_cfg, (
f"Channel '{channel}' from pmaports.cfg in your branch is unknown."
" This appears to be an old branch, consider recreating your change"
" on top of master."
)
return channels_cfg[channel]["branch_pmaports"]
@@ -86,8 +89,15 @@ def get_base_commit() -> str:
return commit
# Add a remote pointing to postmarketOS/pmaports
run_git(["remote", "add", "upstream",
"https://gitlab.postmarketos.org/postmarketOS/pmaports.git"], False)
run_git(
[
"remote",
"add",
"upstream",
"https://gitlab.postmarketos.org/postmarketOS/pmaports.git",
],
False,
)
run_git(["fetch", "-q", "upstream"])
branch_upstream = f"upstream/{get_upstream_branch()}"
@@ -109,10 +119,10 @@ def get_base_commit() -> str:
def get_changed_files(removed=False):
""" Get all changed files and print them, as well as the branch and the
commit that was used for the diff.
:param removed: also return removed files (default: True)
:returns: set of changed files
"""Get all changed files and print them, as well as the branch and the
commit that was used for the diff.
:param removed: also return removed files (default: True)
:returns: set of changed files
"""
commit = get_base_commit()
@@ -131,7 +141,9 @@ def get_changed_files(removed=False):
return ret
def get_changed_packages(skip_archived: bool = False, keep_dir: bool = False) -> set[str]:
def get_changed_packages(
skip_archived: bool = False, keep_dir: bool = False
) -> set[str]:
ret = set()
for file in get_changed_files(removed=True):
dirname, filename = os.path.split(file)
@@ -141,11 +153,12 @@ def get_changed_packages(skip_archived: bool = False, keep_dir: bool = False) ->
# * path with a dot (e.g. .ci/, device/.shared-patches/)
# * documentation
if (
not dirname
or file.startswith(".")
or "/." in file
or skip_archived and dirname.startswith("device/archived")
or dirname == "docs"
not dirname
or file.startswith(".")
or "/." in file
or skip_archived
and dirname.startswith("device/archived")
or dirname == "docs"
):
continue
@@ -185,6 +198,7 @@ def get_changed_kernels(skip_archived: bool = False):
ret += [pkgname]
return ret
def get_changed_devices(skip_archived: bool = False):
ret = []
for pkgname in get_changed_packages(skip_archived):
.ci/lib/deviceinfo_linting.py
+6 -3
View File
@@ -12,7 +12,7 @@ import common
if __name__ == "__main__":
if common.commit_message_has_string("[ci:skip-dint]"):
print("WARNING: not linting deviceinfo files ([ci:skip-dint])")
exit(0)
sys.exit(0)
if "deviceinfo_schema.toml" in common.get_changed_files():
print("deviceinfo_schema.toml changed -> checking all deviceinfo files")
@@ -20,8 +20,11 @@ if __name__ == "__main__":
deviceinfo_files = set(glob.glob(pmaports_dir + "/device/*/*/deviceinfo"))
else:
# only lint deviceinfo files in the devices repo
deviceinfo_files = {file for file in common.get_changed_files()
if os.path.basename(file) == "deviceinfo" and file.startswith("device/")}
deviceinfo_files = {
file
for file in common.get_changed_files()
if os.path.basename(file) == "deviceinfo" and file.startswith("device/")
}
try:
subprocess.run(["dint", "check", *deviceinfo_files], text=True, check=True)
.ci/lib/generate_build_jobs.py
+21 -13
View File
@@ -38,7 +38,7 @@ class Device:
fragment_path = self.full_path / "gitlab-ci.yml.j2"
try:
fragment_tmpl = fragment_path.read_text()
except Exception:
except FileNotFoundError:
return None
return Template(fragment_tmpl).render(device=self)
@@ -47,8 +47,10 @@ class Device:
def pmaports_path(self) -> Path:
# Find the root of pmaports
pmaports_root = self.full_path.parent
while (not (pmaports_root / "pmaports.cfg").exists() and
pmaports_root != pmaports_root.root):
while (
not (pmaports_root / "pmaports.cfg").exists()
and pmaports_root != pmaports_root.root
):
pmaports_root = pmaports_root.parent
assert pmaports_root != pmaports_root.root
@@ -60,11 +62,11 @@ class Device:
@property
def pkgname(self) -> str:
return self.apkbuild['pkgname']
return self.apkbuild["pkgname"]
@property
def arch(self) -> Arch:
return Arch(self.apkbuild['arch'][0])
return Arch(self.apkbuild["arch"][0])
@property
def testing_dependencies(self) -> set[str]:
@@ -88,7 +90,7 @@ class Device:
kernels = []
subpackage_prefix = f"device-{self.codename}-kernel-"
for subpkgname in self.apkbuild.get('subpackages', []):
for subpkgname in self.apkbuild.get("subpackages", []):
if not subpkgname.startswith(subpackage_prefix):
continue
kernel_name = subpkgname.removeprefix(subpackage_prefix)
@@ -129,7 +131,7 @@ class Device:
continue
supported_devices[dev.pmaports_path] = dev
except Exception:
except FileNotFoundError:
traceback.print_exc()
return supported_devices
@@ -138,17 +140,23 @@ class ArchTagSet(set):
def update(self, iterable):
supported_arches = Arch.supported_binary()
# This ignores things like !armv7, that could be a follow-up optimization
if 'noarch' in iterable or 'all' in iterable:
if "noarch" in iterable or "all" in iterable:
iterable = [arch for arch in supported_arches]
super().update([Arch(arch) for arch in iterable if Arch(arch) in supported_arches])
super().update(
[Arch(arch) for arch in iterable if Arch(arch) in supported_arches]
)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
parser.add_argument("--template", default=".ci/build-jobs.yaml.j2", help="Jinja2 input template")
parser.add_argument("--output", default=".ci/build-jobs.yaml", help="output pipeline")
parser.add_argument(
"--template", default=".ci/build-jobs.yaml.j2", help="Jinja2 input template"
)
parser.add_argument(
"--output", default=".ci/build-jobs.yaml", help="output pipeline"
)
args = parser.parse_args()
# pmb logging has to be initialized for later pmb commands to work, setting
@@ -179,13 +187,13 @@ if __name__ == "__main__":
if path.name != "APKBUILD":
continue
apkbuild = pmb.parse.apkbuild(path)
packages_modified.add(apkbuild['pkgname'])
packages_modified.add(apkbuild["pkgname"])
archs.update(apkbuild["arch"])
# Add all the devices found in CI that depend on the package that got
# modified
for device in supported_devices.values():
if apkbuild['pkgname'] in device.dependencies:
if apkbuild["pkgname"] in device.dependencies:
devices_under_test.add(device)
if common.commit_message_has_string("[ci:skip-build]"):
.ci/lib/kconfig.py
+14 -10
View File
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import subprocess
import sys
# Same dir
import common
@@ -12,10 +13,11 @@ def check_kconfig(pkgnames):
last_failed = None
for i in range(len(pkgnames)):
pkgname = pkgnames[i]
print(f" ({i+1}/{len(pkgnames)}) {pkgname}")
print(f" ({i + 1}/{len(pkgnames)}) {pkgname}")
p = subprocess.run(["pmbootstrap", "kconfig", "check", "--keep-going", pkgname],
check=False)
p = subprocess.run(
["pmbootstrap", "kconfig", "check", "--keep-going", pkgname], check=False
)
if p.returncode:
last_failed = pkgname
@@ -32,8 +34,10 @@ def show_error(last_failed):
print()
print("---")
print()
print("Please adjust your kernel config. This is required for getting your"
" patch merged.")
print(
"Please adjust your kernel config. This is required for getting your"
" patch merged."
)
print()
print("Edit your kernel config:")
print(f" pmbootstrap kconfig edit {last_failed}")
@@ -76,21 +80,21 @@ if __name__ == "__main__":
print("kconfigcheck.toml changed -> checking all kernels")
if not check_kconfig_all():
show_error_all()
exit(1)
exit(0)
sys.exit(1)
sys.exit(0)
pkgnames = common.get_changed_kernels(skip_archived=True)
print(f"Changed kernels: {pkgnames}")
if len(pkgnames) == 0:
print("No kernels changed in this branch")
exit(0)
sys.exit(0)
elif common.commit_message_has_string("[ci:skip-kconfigcheck]"):
print("WARNING: not checking kernel configs ([ci:skip-kconfigcheck])")
exit(0)
sys.exit(0)
last_failed = check_kconfig(pkgnames)
if last_failed:
show_error(last_failed)
exit(1)
sys.exit(1)
.ci/lib/verify_checksums.py
+6 -5
View File
@@ -19,8 +19,9 @@ def verify_checksums(packages):
print("verifying checksums: " + ", ".join(packages))
common.run_pmbootstrap(["build_init"])
common.run_pmbootstrap(["--details-to-stdout", "checksum", "--verify"] +
list(packages))
common.run_pmbootstrap(
["--details-to-stdout", "checksum", "--verify"] + list(packages)
)
if __name__ == "__main__":
@@ -39,14 +40,14 @@ if __name__ == "__main__":
archived = set()
for path in pmb.core.pkgrepo.pkgrepo_iter_package_dirs():
# Store archived packages for matching later
if os.path.split(os.path.split(path)[0])[1] == 'archived':
if os.path.split(os.path.split(path)[0])[1] == "archived":
archived.add(os.path.basename(path))
continue
buildable_pkgs.add(os.path.basename(path))
# To store a list of packages from extra-repos/systemd for special handling
# later:
systemd_pkgs = list()
systemd_pkgs = []
# Filter out packages are not found in enabled repos or are in archived
# (Iterate over copy of packages, because we modify it in this loop)
@@ -73,7 +74,7 @@ if __name__ == "__main__":
if len(packages) == 0:
print("no packages to check for the current repo")
else:
verify_checksums(packages)
verify_checksums(packages)
# FIXME: this should probably be more generic, if other repos are added later?
if systemd_pkgs:
.ci/ruff.sh
+1 -1
View File
@@ -11,4 +11,4 @@ fi
set -x
# shellcheck disable=SC2046
ruff check $(find .ci -name '*.py')
ruff check --ignore LOG015 $(find .ci -name '*.py')
.ci/testcases/test_device.py
+31 -21
View File
@@ -15,13 +15,15 @@ gitlab_groups = [
def device_dependency_check(apkbuild, path):
""" Raise an error if a device package has a dependency that is not allowed
(e.g. because it should be in a subpackage instead). """
"""Raise an error if a device package has a dependency that is not allowed
(e.g. because it should be in a subpackage instead)."""
for depend in apkbuild["depends"]:
if depend == "mesa-dri-gallium":
raise RuntimeError(f"{path}: mesa-dri-gallium shouldn't be in"
" depends anymore (see pmaports!3478)")
raise RuntimeError(
f"{path}: mesa-dri-gallium shouldn't be in"
" depends anymore (see pmaports!3478)"
)
def test_aports_device():
@@ -44,11 +46,12 @@ def test_aports_device():
device_dependency_check(apkbuild, path)
# Architecture
device = apkbuild["pkgname"][len("device-"):]
device = apkbuild["pkgname"][len("device-") :]
deviceinfo = pmb.parse.deviceinfo(device)
if Arch.from_str("".join(apkbuild["arch"])) != deviceinfo.arch:
raise RuntimeError("wrong architecture, please change to arch=\""
f"{deviceinfo.arch}\": {path}")
raise RuntimeError(
f'wrong architecture, please change to arch="{deviceinfo.arch}": {path}'
)
if "!archcheck" not in apkbuild["options"]:
raise RuntimeError(f"!archcheck missing in options= line: {path}")
@@ -64,7 +67,7 @@ def test_aports_device_kernel():
for path in aports.glob("device/*/device-*/APKBUILD"):
# Parse apkbuild and kernels from subpackages
apkbuild = pmb.parse.apkbuild(path)
device = apkbuild["pkgname"][len("device-"):]
device = apkbuild["pkgname"][len("device-") :]
kernels_subpackages = pmb.parse._apkbuild.kernels(device)
# Parse kernels from depends
@@ -76,20 +79,24 @@ def test_aports_device_kernel():
# Kernel in subpackages *and* depends
if kernels_subpackages:
raise RuntimeError(f"Kernel package '{depend}' needs to"
" be removed when using kernel" +
f" subpackages: {path}")
raise RuntimeError(
f"Kernel package '{depend}' needs to"
" be removed when using kernel" + f" subpackages: {path}"
)
# No kernel
if not kernels_depends and not kernels_subpackages:
raise RuntimeError("Device doesn't have a kernel in depends or"
f" subpackages: {path}")
raise RuntimeError(
f"Device doesn't have a kernel in depends or subpackages: {path}"
)
# Multiple kernels in depends
if len(kernels_depends) > 1:
raise RuntimeError("Please use kernel subpackages instead of"
" multiple kernels in depends (see"
f" <https://postmarketos.org/devicepkg>): {path}")
raise RuntimeError(
"Please use kernel subpackages instead of"
" multiple kernels in depends (see"
f" <https://postmarketos.org/devicepkg>): {path}"
)
# @pytest.mark.xfail # Not all aports have been updated yet
@@ -100,14 +107,15 @@ def test_aports_maintained():
"""
for path in pkgrepo_iglob("device/main/*/APKBUILD"):
if 'firmware-' in path.parent.name:
if "firmware-" in path.parent.name:
continue
maintainers = pmb.parse._apkbuild.maintainers(path)
assert maintainers and len(maintainers) >= 2, \
assert maintainers and len(maintainers) >= 2, (
f"{path} in main needs at least 1 Maintainer and 1 Co-Maintainer"
)
for path in pkgrepo_iglob("device/community/*/APKBUILD"):
if 'firmware-' in path.parent.name:
if "firmware-" in path.parent.name:
continue
maintainers = pmb.parse._apkbuild.maintainers(path)
assert maintainers, f"{path} in community needs at least 1 Maintainer"
@@ -121,5 +129,7 @@ def test_aports_archived():
for path in pkgrepo_iglob("device/archived/*/APKBUILD"):
archived = pmb.parse._apkbuild.archived(path)
assert archived, f"{path} should have an Archived: " +\
"comment that describes why the package is archived"
assert archived, (
f"{path} should have an Archived: "
+ "comment that describes why the package is archived"
)
.ci/testcases/test_deviceinfo.py
+19 -15
View File
@@ -14,14 +14,13 @@ def test_deviceinfo():
# Iterate over all devices
last_exception = None
count = 0
pattern = re.compile("^deviceinfo_[a-zA-Z0-9_]*=\".*\"(\\s*# .*)?$")
pattern = re.compile('^deviceinfo_[a-zA-Z0-9_]*=".*"(\\s*# .*)?$')
for folder in pkgrepo_iglob("device/*/device-*"):
device = folder.name.split("-", 1)[1]
f = open(folder / "deviceinfo")
lines = f.read().split("\n")
f.close()
with open(folder / "deviceinfo") as f:
lines = f.read().split("\n")
try:
for line in lines:
@@ -30,10 +29,12 @@ def test_deviceinfo():
continue
# Check line against regex (can't use multiple lines etc.)
if not pattern.match(line) or line.endswith("\\\""):
raise RuntimeError("Line looks invalid, maybe missing"
" quotes/multi-line string/malformed"
f" inline comment? {line}")
if not pattern.match(line) or line.endswith('\\"'):
raise RuntimeError(
"Line looks invalid, maybe missing"
" quotes/multi-line string/malformed"
f" inline comment? {line}"
)
# Successful deviceinfo parsing / obsolete options
info = pmb.parse.deviceinfo(device)
@@ -41,15 +42,18 @@ def test_deviceinfo():
# deviceinfo_name must start with manufacturer
name = info.name
manufacturer = info.manufacturer
if not name.startswith(manufacturer) and \
not name.startswith("Google") and \
not name.startswith("Jolla"):
raise RuntimeError("Please add the manufacturer in front of"
" the deviceinfo_name, e.g.: '" +
manufacturer + " " + name + "'")
if (
not name.startswith(manufacturer)
and not name.startswith("Google")
and not name.startswith("Jolla")
):
raise RuntimeError(
"Please add the manufacturer in front of"
" the deviceinfo_name, e.g.: '" + manufacturer + " " + name + "'"
)
# Don't abort on first error
except Exception as e:
except (RuntimeError, SyntaxError, ValueError) as e:
last_exception = e
count += 1
print(device + ": " + str(e))
.ci/testcases/test_directory_structure.py
+20 -14
View File
@@ -23,9 +23,11 @@ expected_directories = [
# This test makes sure all of them are in one of the expected locations.
def test_directories():
apkbuilds = set(glob.iglob("**/APKBUILD", recursive=True))
expected = set(f for d in expected_directories for f in glob.iglob(d + "/*/APKBUILD"))
assert apkbuilds == expected, "Found APKBUILD in unexpected directory. " \
expected = {f for d in expected_directories for f in glob.iglob(d + "/*/APKBUILD")}
assert apkbuilds == expected, (
"Found APKBUILD in unexpected directory. "
"Note that we moved firmware/* to device/{main,community,testing}/*."
)
# Ensure no file in pmaports are executable.
@@ -38,11 +40,13 @@ def test_executable_files():
permissions = os.stat(file).st_mode
executable_bits = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
if permissions & executable_bits != 0:
raise RuntimeError(f"\"{file}\" is executable. Files in pmaports" +
" should not be executables. post-* files" +
" don't need to be executable and executables" +
" should be installed using `install -D" +
"m0755` or a variation thereof.")
raise RuntimeError(
f'"{file}" is executable. Files in pmaports'
+ " should not be executables. post-* files"
+ " don't need to be executable and executables"
+ " should be installed using `install -D"
+ "m0755` or a variation thereof."
)
# Make sure files belong to a package (below a directory with APKBUILD)
@@ -54,24 +58,26 @@ def test_files_belong_to_package():
# Skip "hidden" directories (.ci/, device/.shared-patches/)
dirs[:] = [d for d in dirs if not d.startswith(".")]
# Ignore files in root directory (README.md)
if dirpath == '.':
if dirpath == ".":
continue
# Ignore files under docs/
if dirpath.startswith('./docs'):
if dirpath.startswith("./docs"):
continue
# Switched to another directory?
if package_dir and not dirpath.startswith(package_dir + os.sep):
package_dir = None
if 'APKBUILD' in files:
assert not package_dir, f"Nested packages: {package_dir} and {dirpath} " \
"both contain an APKBUILD"
if "APKBUILD" in files:
assert not package_dir, (
f"Nested packages: {package_dir} and {dirpath} both contain an APKBUILD"
)
package_dir = dirpath
# Skip specific ".gitlab-ci.yml" files outside of usual structure
if dirpath in ["./cross"]:
files[:] = [f for f in files if f != ".gitlab-ci.yml"]
assert not files or package_dir, "Found files that do not belong to any package: " \
f"{dirpath}/{files}"
assert not files or package_dir, (
f"Found files that do not belong to any package: {dirpath}/{files}"
)
.ci/testcases/test_firmware.py
+20 -15
View File
@@ -22,28 +22,33 @@ def test_aports_firmware():
"firmware-samsung-maguro", # Depends on firmware-aosp-broadcom-wlan
"firmware-xiaomi-ferrari", # Depends on soc-qcom-msm8916
"firmware-xiaomi-willow", # Doesn't build, source link is dead (pma#1212)
"firmware-samsung-i9105p", # Depends on firmware-aosp-broadcom-wlan
"firmware-samsung-i9105p", # Depends on firmware-aosp-broadcom-wlan
]
for path in pkgrepo_iglob("**/firmware-*/APKBUILD", recursive=True):
apkbuild = pmb.parse.apkbuild(path)
aport_name = os.path.basename(path.parent)
if aport_name not in excluded:
if "pmb:cross-native" not in apkbuild["options"]:
raise RuntimeError(f"{aport_name}: \"pmb:cross-native\" missing in"
" options= line. The pmb:cross-native option is"
" preferred because it results in significantly"
" lower build times. If the package doesn't build"
" with the option, you can add an exemption in"
" .gitlab-ci/testcases/test_firmware.py.")
if aport_name not in excluded and "pmb:cross-native" not in apkbuild["options"]:
raise RuntimeError(
f'{aport_name}: "pmb:cross-native" missing in'
" options= line. The pmb:cross-native option is"
" preferred because it results in significantly"
" lower build times. If the package doesn't build"
" with the option, you can add an exemption in"
" .gitlab-ci/testcases/test_firmware.py."
)
if "!tracedeps" not in apkbuild["options"]:
raise RuntimeError(f"{aport_name}: \"!tracedeps\" missing in"
" options= line. The tracedeps option is superfluous"
" for firmware packages.")
raise RuntimeError(
f'{aport_name}: "!tracedeps" missing in'
" options= line. The tracedeps option is superfluous"
" for firmware packages."
)
if "noarch" in apkbuild["arch"]:
raise RuntimeError(f"{aport_name}: \"arch\" must not be \"noarch\"!"
" Please limit this firmware package to the"
" required architectures only!")
raise RuntimeError(
f'{aport_name}: "arch" must not be "noarch"!'
" Please limit this firmware package to the"
" required architectures only!"
)
.ci/testcases/test_framework_versions.py
+8 -10
View File
@@ -1,9 +1,9 @@
# Copyright 2021 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
""" Make sure that components of frameworks (Qt, KDE, ...) have the same
version in all packages. We scan all packages, categorize them by URL
and then compare the versions of all packages in each category. """
"""Make sure that components of frameworks (Qt, KDE, ...) have the same
version in all packages. We scan all packages, categorize them by URL
and then compare the versions of all packages in each category."""
import logging
@@ -41,8 +41,7 @@ def get_categorized_packages():
category = "plasma"
elif "https://community.kde.org/Frameworks" in url:
category = "kde"
elif url in ["http://qt-project.org/",
"https://www.qt.io/developers/"]:
elif url in ["http://qt-project.org/", "https://www.qt.io/developers/"]:
category = "qt"
# Remove hotfix number (i.e. 5.16.90.1 becomes 5.16.90)
@@ -69,7 +68,6 @@ def check_categories(categories):
for category, packages in categories.items():
reference = None
for pkgname, pkgver in packages.items():
# Use the first package as reference and print a summary
if not reference:
logging.info("---")
@@ -87,8 +85,9 @@ def check_categories(categories):
# Print the reference and skip checking it against itself
reference = {"pkgname": pkgname, "pkgver": pkgver}
logging.info("Reference pkgver: " + pkgver + " (from '" +
pkgname + "')")
logging.info(
"Reference pkgver: " + pkgver + " (from '" + pkgname + "')"
)
continue
# Check version against reference
@@ -98,8 +97,7 @@ def check_categories(categories):
# Each category must at least have one package
if not reference:
logging.info("ERROR: could not find any packages in category: " +
category)
logging.info("ERROR: could not find any packages in category: " + category)
ret = False
return ret
.ci/testcases/test_kernel.py
+36 -17
View File
@@ -21,29 +21,48 @@ def test_aports_kernel():
continue # This package isn't a linux kernel!
if "pmb:cross-native" not in apkbuild["options"]:
raise RuntimeError(f"{aport_name}: \"pmb:cross-native\" missing in"
" options= line")
raise RuntimeError(
f'{aport_name}: "pmb:cross-native" missing in options= line'
)
# cross-compilers should not be in makedepends
for ccc in ["gcc-armv7", "gcc-armhf", "gcc-aarch64",
"gcc4-armv7", "gcc4-armhf", "gcc4-aarch64",
"gcc6-armv7", "gcc6-armhf", "gcc6-aarch64"]:
for ccc in [
"gcc-armv7",
"gcc-armhf",
"gcc-aarch64",
"gcc4-armv7",
"gcc4-armhf",
"gcc4-aarch64",
"gcc6-armv7",
"gcc6-armhf",
"gcc6-aarch64",
]:
if ccc in apkbuild["makedepends"]:
raise RuntimeError(f"{aport_name}: Cross-compiler ({ccc}) should"
" not be explicitly specified in makedepends!"
" pmbootstrap installs cross-compiler"
" automatically.")
raise RuntimeError(
f"{aport_name}: Cross-compiler ({ccc}) should"
" not be explicitly specified in makedepends!"
" pmbootstrap installs cross-compiler"
" automatically."
)
# check some options only for main and community devices
_, relpath = pkgrepo_relative_path(path)
for part in ["main", "community"]:
if part in relpath.parts:
if "pmb:kconfigcheck-community" not in apkbuild["options"]:
raise RuntimeError(f"{aport_name}: \"pmb:kconfigcheck-community\" missing in"
" options= line, required for all community/main devices.")
if (
part in relpath.parts
and "pmb:kconfigcheck-community" not in apkbuild["options"]
):
raise RuntimeError(
f'{aport_name}: "pmb:kconfigcheck-community" missing in'
" options= line, required for all community/main devices."
)
# check for postmarketos-installkernel in makedepends when installing kernel with make
if bool(re.search("make z?install", path.read_text(encoding="utf-8"))):
if "postmarketos-installkernel" not in apkbuild["makedepends"]:
raise RuntimeError(f"{aport_name}: \"postmarketos-installkernel\" missing in"
" makedepends, required when using make install/zinstall.")
if (
bool(re.search("make z?install", path.read_text(encoding="utf-8")))
and "postmarketos-installkernel" not in apkbuild["makedepends"]
):
raise RuntimeError(
f'{aport_name}: "postmarketos-installkernel" missing in'
" makedepends, required when using make install/zinstall."
)
.ci/testcases/test_provides.py
+9 -4
View File
@@ -44,7 +44,9 @@ def apkbuild_check_provides(path, apkbuild, version, pkgname, subpkgname=None):
if not provide.endswith(f"={version}"):
# Valid version strings, per the APKBUILD reference, are fine. This regex
# attempts to detect those.
pattern = r'^\d+(\.\d+)*[a-z]?(_(?:alpha|beta|pre|rc|cvs|svn|git|hg|p)\d*)*-r\d+$'
pattern = (
r"^\d+(\.\d+)*[a-z]?(_(?:alpha|beta|pre|rc|cvs|svn|git|hg|p)\d*)*-r\d+$"
)
[provide_no_ver, provide_ver] = provide.split("=", 1)
if not re.match(pattern, provide_ver):
error = f"error in , provide_ver{path}:\n"
@@ -62,7 +64,9 @@ def apkbuild_check_provides(path, apkbuild, version, pkgname, subpkgname=None):
error += " version on purpose, you also need to set a"
error += " provider_priority (pma#1766).\n"
error += "Reference:"
error += " https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#provides"
error += (
" https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#provides"
)
error += f"\nVERSION: {provide_no_ver}"
ret += [error]
return ret
@@ -82,8 +86,9 @@ def test_provides():
if not subpkg_data:
# default packaging function like -doc
continue
errors += apkbuild_check_provides(path_rel, subpkg_data, version,
pkgname, subpkg)
errors += apkbuild_check_provides(
path_rel, subpkg_data, version, pkgname, subpkg
)
if errors:
for error in errors:
.ci/testcases/test_source.py
+30 -16
View File
@@ -45,9 +45,11 @@ def parse_source_from_checksums(apkbuild_path):
try:
checksum, filename = line.rstrip().split(" ", 2)
except ValueError:
raise ValueError("Failed to parse checksums. Try to delete the"
" checksums and generate them again with"
f" 'pmbootstrap checksum': {apkbuild_path}")
raise ValueError(
"Failed to parse checksums. Try to delete the"
" checksums and generate them again with"
f" 'pmbootstrap checksum': {apkbuild_path}"
)
# Cut off 'sha512sums="' if the first checksum is in that line
checksum = checksum.removeprefix(start)
@@ -96,15 +98,21 @@ def test_aports_unreferenced_files():
if rel_file_path in ["APKBUILD", "gitlab-ci.yml.j2"] or os.path.isdir(file):
continue
# Skip kernel fragments in linux-* packages
if os.path.basename(rel_file_path).endswith(".config") and apkbuild["pkgname"].startswith("linux-"):
if os.path.basename(rel_file_path).endswith(".config") and apkbuild[
"pkgname"
].startswith("linux-"):
continue
if os.path.basename(rel_file_path) not in sources_chk \
and rel_file_path not in apkbuild["install"] \
and rel_file_path not in subpackage_installs \
and rel_file_path not in trigger_sources \
and not rel_file_path.startswith("tests/"):
raise RuntimeError(f"{apkbuild_path}: found unreferenced file: {rel_file_path}")
if (
os.path.basename(rel_file_path) not in sources_chk
and rel_file_path not in apkbuild["install"]
and rel_file_path not in subpackage_installs
and rel_file_path not in trigger_sources
and not rel_file_path.startswith("tests/")
):
raise RuntimeError(
f"{apkbuild_path}: found unreferenced file: {rel_file_path}"
)
def test_distfiles_conflict():
@@ -131,8 +139,10 @@ def test_distfiles_conflict():
# First time seeing this file
if filename not in source_all:
source_all[filename] = {"checksum": checksum,
"apkbuild_rel": apkbuild_rel}
source_all[filename] = {
"checksum": checksum,
"apkbuild_rel": apkbuild_rel,
}
continue
# Saw this file already with same checksum
@@ -141,15 +151,19 @@ def test_distfiles_conflict():
# Saw this file already with different checksum
logging.error("")
logging.error(f"ERROR: the source file '{filename}' has different"
" checksums in the following files:")
logging.error(
f"ERROR: the source file '{filename}' has different"
" checksums in the following files:"
)
logging.error(f"- {source_all[filename]['apkbuild_rel']}:")
logging.error(f" {source_all[filename]['checksum']}")
logging.error(f"- {apkbuild_rel}:")
logging.error(f" {checksum}")
logging.error("")
logging.error("Fix this by setting a different target filename in"
" the package you modified:")
logging.error(
"Fix this by setting a different target filename in"
" the package you modified:"
)
logging.error("https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#source")
logging.error("")
raise RuntimeError(f"Conflict with source file '{filename}'")
.ci/testcases/test_ui.py
+8 -4
View File
@@ -31,8 +31,10 @@ def test_aports_ui():
for package in apkbuild["_pmb_recommends"]:
depend = pmb.helpers.package.get(package, arch, must_exist=False)
if depend is None:
raise RuntimeError(f"{path}: package '{package}' from"
f" _pmb_recommends not found for arch '{arch}'")
raise RuntimeError(
f"{path}: package '{package}' from"
f" _pmb_recommends not found for arch '{arch}'"
)
# Check packages from "_pmb_recommends" of -extras subpackage if one exists
if f"{apkbuild['pkgname']}-extras" in apkbuild["subpackages"]:
@@ -40,5 +42,7 @@ def test_aports_ui():
for package in apkbuild["_pmb_recommends"]:
depend = pmb.helpers.package.get(package, arch, must_exist=False)
if depend is None:
raise RuntimeError(f"{path}: package '{package}' from _pmb_recommends "
f"of -extras subpackage is not found for arch '{arch}'")
raise RuntimeError(
f"{path}: package '{package}' from _pmb_recommends "
f"of -extras subpackage is not found for arch '{arch}'"
)