ci: build packages in a child pipeline instead of in the main one (MR 6366)

Following Alpine's example. This allows some optimizations:
* Don't start one job per architecture simply to skip them after
  rebase and before merge
* If there are no packages, don't start the jobs at all (not true,
  gitlab bug)
* Avoid creating jobs for architectures that don't have changed
  packages, thus further reducing the unnecessary
  jobs and possibly reducing transient failures

Co-authored-by: Clayton Craft <clayton@craftyguy.net>

.ci/build-jobs.yaml.j2

@@ -0,0 +1,80 @@
+image: alpine:latest
+stages:
+  - build
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "parent_pipeline"
+
+{% if archs|length == 0 %}
+# Needed because gitlab fails if no jobs are created, e.g: nothing should be
+# built: https://gitlab.com/gitlab-org/gitlab/-/issues/368248
+placeholder:
+  stage: build
+  script:
+    - 'true'
+{% endif %}
+
+.build:
+  stage: build
+  interruptible: true
+  before_script:
+    # upgrade is mostly needed for "qemu" runners that run edge based on a VM
+    # The other build jobs run natively in docker on latest alpine stable and
+    # those should always have whatever we need
+    - apk upgrade -U
+    - .ci/lib/gitlab_prepare_ci.sh
+  after_script:
+    - cp -r /home/pmos/.local/var/pmbootstrap/packages/ packages/ || true
+    - .ci/lib/move_logs.sh $CI_PROJECT_DIR
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - packages/
+  timeout: 10 h
+
+{% if ('x86_64' in archs) %}
+build-x86_64:
+  extends: .build
+  script:
+    - .ci/build-x86_64.sh
+{% endif %}
+
+{% if ('x86' in archs) %}
+build-x86:
+  extends: .build
+  script:
+    - .ci/build-x86.sh
+{% endif %}
+
+{% if ('aarch64' in archs) %}
+build-aarch64:
+  extends: .build
+  tags: [arm64]
+  script:
+    - .ci/build-aarch64.sh
+{% endif %}
+
+{% if ('armv7' in archs) %}
+build-armv7:
+  extends: .build
+  tags: [qemu]
+  script:
+    - .ci/build-armv7.sh
+{% endif %}
+
+{% if ('armhf' in archs) %}
+build-armhf:
+  extends: .build
+  tags: [qemu]
+  script:
+    - .ci/build-armhf.sh
+{% endif %}
+
+{% if ('riscv64' in archs) %}
+build-riscv64:
+  extends: .build
+  tags: [qemu]
+  script:
+    - .ci/build-riscv64.sh
+{% endif %}

.ci/lib/generate_build_jobs.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+# Copyright 2025 Pablo Correa Gomez
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from pathlib import Path
+import shutil
+import sys
+
+from jinja2 import Template
+import add_pmbootstrap_to_import_path
+import pmb.parse
+import pmb.helpers.logging
+from pmb.core.arch import Arch
+
+# Same dir
+import common
+
+if __name__ == "__main__":
+    # Needs input to output if we should create the jobs
+    if len(sys.argv) != 3:
+        print("usage: generate_build_jobs.py TEMPLATE CHILD_PIPELINE")
+        print(sys.argv)
+        sys.exit(1)
+    template = sys.argv[1]
+    child_pipeline = sys.argv[2]
+
+    # pmb logging has to be initialized for later pmb commands to work, setting
+    # to /dev/null since we don't care about the output. Later this could
+    # be changed to a file and added as a CI artifact if we need to debug
+    # something.
+    pmb.logging.init(Path("/dev/null"), False)
+    archs = set()
+    # Get and print modified packages
+    common.add_upstream_git_remote()
+    for file in common.get_changed_files():
+        path = Path(file)
+        if path.name != "APKBUILD":
+            continue
+        elif not path.exists():
+            continue  # APKBUILD was deleted
+        apkbuild = pmb.parse.apkbuild(path)
+        archs.update(apkbuild["arch"])
+
+    if common.commit_message_has_string("[ci:skip-build]"):
+        print("User requested skipping build, not creating child pipeline file")
+        archs = set()
+
+    # This ignores things like !armv7, that could be a follow-up optimization
+    if 'noarch' in archs or 'all' in archs:
+        archs = set([str(arch) for arch in Arch.supported()])
+
+    print(archs)
+    with open(template) as f:
+        rendered = Template(f.read()).render(archs=archs)
+        with open(child_pipeline, "w") as fw:
+            fw.write(rendered)

.gitignore

@@ -118,3 +118,6 @@ ENV/
 
 # Allow having untracked packages in aports using a custom-something directory
 /custom-*/
+
+# child pipeline
+.ci/build-jobs.yaml

.gitlab-ci.yml

@@ -8,6 +8,7 @@ after_script:
   - .ci/lib/move_logs.sh $CI_PROJECT_DIR
 stages:
   - lint
+  - prepare
   - build
   - autoupdate
   - test
@@ -142,55 +143,44 @@ mr-settings:
     - wget -q "https://gitlab.postmarketos.org/postmarketOS/ci-common/-/raw/master/check_mr_settings.py"
     - python3 ./check_mr_settings.py
 
-# build changed aports
+# Calculate build of changed aports using a dynamic downstream pipelines:
-.build:
+# https://docs.gitlab.com/ci/pipelines/downstream_pipelines/#dynamic-child-pipelines
-  stage: build
+# For those to work, we need two jobs:
+# * First one (this one) that generates a valid yaml file with the job yaml as
+#   an artifact
+# * Second one (build-jobs) that has a "trigger" key with the yaml file from
+#   which to trigger pipelines
+# To avoid reworking the whole CI file, for now we are re-using this same file
+# to trigger the downstream pipeline. In that case, $CI_PIPELINE_SOURCE will
+# be set to "parent_pipeline", and only the jobs that extend .build will run
+generate-build-jobs:
+  stage: prepare
+  before_script:
+    - .ci/lib/gitlab_prepare_ci.sh
+  needs: []
+  script:
+    - wget "https://gitlab.postmarketos.org/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
+    - sh ./install_pmbootstrap.sh py3-jinja2
+    - .ci/lib/generate_build_jobs.py .ci/build-jobs.yaml.j2 .ci/build-jobs.yaml
+
+  artifacts:
+    paths:
+      - .ci/build-jobs.yaml
   rules:
     - if: *mr_rule
-  before_script:
-    - *global_before_scripts
-    - .ci/lib/gitlab_prepare_ci.sh
-  after_script:
-    - cp -r /home/pmos/.local/var/pmbootstrap/packages/ packages/ || true
-  artifacts:
-    expire_in: 1 week
-    paths:
-      - packages/
-  timeout: 10 h
 
-build-x86_64:
+build-jobs:
-  extends: .build
+  stage: build
-  script:
+  trigger:
-    - .ci/build-x86_64.sh
+    include:
-
+      - artifact: .ci/build-jobs.yaml
-build-x86:
+        job: generate-build-jobs
-  extends: .build
+    strategy: depend
-  script:
+    forward:
-    - .ci/build-x86.sh
+      # So that we don't have to duplicate GIT_DEPTH
-
+      pipeline_variables: true
-build-aarch64:
+  rules:
-  extends: .build
+    - if: *mr_rule
-  tags: [arm64]
-  script:
-    - .ci/build-aarch64.sh
-
-build-armv7:
-  extends: .build
-  tags: [qemu]
-  script:
-    - .ci/build-armv7.sh
-
-build-armhf:
-  extends: .build
-  tags: [qemu]
-  script:
-    - .ci/build-armhf.sh
-
-build-riscv64:
-  extends: .build
-  tags: [qemu]
-  script:
-    - .ci/build-riscv64.sh
 
 auto-update:
   stage: autoupdate