[XEN PATCH 3/3] CI: New stage "containers" to rebuild some containers

["Anthony PERARD" <anthony.perard@xxxxxxxxxx>]

Rebuild rolling release containers when XEN_CI_REBUILD_CONTAINERS is
set. This is to be use with a scheduled pipeline.

When $XEN_CI_REBUILD_CONTAINERS is set, only build jobs related to the
containers been rebuild will be executed.

Build jobs that are using one of the containers been rebuild should
wait for the container to be rebuild. If it's a normal pipeline, those
dependency are simply ignored.

Signed-off-by: Anthony PERARD <anthony.perard@xxxxxxxxxx>
---
 .gitlab-ci.yml                       |  2 +
 automation/gitlab-ci/build.yaml      | 57 ++++++++++++++++++----------
 automation/gitlab-ci/containers.yaml | 29 ++++++++++++++
 automation/gitlab-ci/test.yaml       | 13 +++++++
 4 files changed, 81 insertions(+), 20 deletions(-)
 create mode 100644 automation/gitlab-ci/containers.yaml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 941e5822e8..dab9171086 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,11 +8,13 @@ workflow:
     - when: always
 
 stages:
+  - containers
   - analyze
   - build
   - test
 
 include:
+  - 'automation/gitlab-ci/containers.yaml'
   - 'automation/gitlab-ci/analyze.yaml'
   - 'automation/gitlab-ci/build.yaml'
   - 'automation/gitlab-ci/test.yaml'
diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
index 1ca6764225..e01eec1423 100644
--- a/automation/gitlab-ci/build.yaml
+++ b/automation/gitlab-ci/build.yaml
@@ -3,6 +3,9 @@
   image: ${XEN_REGISTRY}/${CONTAINER}
   script:
     - ./automation/scripts/build 2>&1 | tee build.log
+  rules:
+    - if: $XEN_CI_REBUILD_CONTAINERS && $XEN_CI_RUN_AFTER_CONTAINER_BUILD
+    - if: $XEN_CI_REBUILD_CONTAINERS == null
   artifacts:
     paths:
       - binaries/
@@ -255,6 +258,8 @@
 .test-jobs-artifact-common:
   stage: build
   needs: []
+  rules:
+    - if: $XEN_CI_REBUILD_CONTAINERS == null
 
 # Arm test artifacts
 
@@ -516,17 +521,24 @@ alpine-3.18-clang-debug:
   variables:
     CONTAINER: alpine:3.18
 
-archlinux-gcc:
-  extends: .gcc-x86-64-build
+.container-archlinux-current:
   variables:
     CONTAINER: archlinux:current
+    XEN_CI_RUN_AFTER_CONTAINER_BUILD: true
+  needs:
+    - job: container-archlinux-current
+      optional: true
   allow_failure: true
 
+archlinux-gcc:
+  extends:
+    - .gcc-x86-64-build
+    - .container-archlinux-current
+
 archlinux-gcc-debug:
-  extends: .gcc-x86-64-build-debug
-  variables:
-    CONTAINER: archlinux:current
-  allow_failure: true
+  extends:
+    - .gcc-x86-64-build-debug
+    - .container-archlinux-current
 
 centos-7-gcc:
   extends: .gcc-x86-64-build
@@ -657,29 +669,34 @@ opensuse-leap-15.6-gcc-debug:
   variables:
     CONTAINER: opensuse:leap-15.6-x86_64
 
-opensuse-tumbleweed-clang:
-  extends: .clang-x86-64-build
+.container-opensuse-tumbleweed-x86_64:
   variables:
     CONTAINER: opensuse:tumbleweed-x86_64
+    XEN_CI_RUN_AFTER_CONTAINER_BUILD: true
+  needs:
+    - job: container-opensuse-tumbleweed-x86_64
+      optional: true
   allow_failure: true
 
+opensuse-tumbleweed-clang:
+  extends:
+    - .clang-x86-64-build
+    - .container-opensuse-tumbleweed-x86_64
+
 opensuse-tumbleweed-clang-debug:
-  extends: .clang-x86-64-build-debug
-  variables:
-    CONTAINER: opensuse:tumbleweed-x86_64
-  allow_failure: true
+  extends:
+    - .clang-x86-64-build-debug
+    - .container-opensuse-tumbleweed-x86_64
 
 opensuse-tumbleweed-gcc:
-  extends: .gcc-x86-64-build
-  variables:
-    CONTAINER: opensuse:tumbleweed-x86_64
-  allow_failure: true
+  extends:
+    - .gcc-x86-64-build
+    - .container-opensuse-tumbleweed-x86_64
 
 opensuse-tumbleweed-gcc-debug:
-  extends: .gcc-x86-64-build-debug
-  variables:
-    CONTAINER: opensuse:tumbleweed-x86_64
-  allow_failure: true
+  extends:
+    - .gcc-x86-64-build-debug
+    - .container-opensuse-tumbleweed-x86_64
 
 # PowerPC builds (x86 cross)
 debian-11-ppc64le-gcc:
diff --git a/automation/gitlab-ci/containers.yaml 
b/automation/gitlab-ci/containers.yaml
new file mode 100644
index 0000000000..25e8bdc34b
--- /dev/null
+++ b/automation/gitlab-ci/containers.yaml
@@ -0,0 +1,29 @@
+.container-build-tmpl:
+  stage: containers
+  image: docker:stable
+  tags:
+    - container-builder
+  rules:
+    - if: $XEN_CI_REBUILD_CONTAINERS
+  services:
+    - docker:dind
+  before_script:
+    - apk add make
+    - docker info
+    - docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY
+  script:
+    - make -C automation/build PUSH=1 REGISTRY=${XEN_REGISTRY} 
${CONTAINER/:/\/}
+  after_script:
+    - docker logout
+
+container-archlinux-current:
+  extends:
+    - .container-build-tmpl
+  variables:
+    CONTAINER: "archlinux:current"
+
+container-opensuse-tumbleweed-x86_64:
+  extends:
+    - .container-build-tmpl
+  variables:
+    CONTAINER: "opensuse:tumbleweed-x86_64"
diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index 1822e3ea5f..e4d4f1f3c9 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -1,6 +1,16 @@
+.test-rules:
+  # Only add rules which prevent jobs from been added to the pipeline. They
+  # should all have "when: never".
+  pre_rules:
+    - if: $XEN_CI_REBUILD_CONTAINERS
+      when: never
+
 .test-jobs-common:
   stage: test
   image: ${XEN_REGISTRY}/${CONTAINER}
+  rules:
+    - !reference [.test-rules, pre_rules]
+    - when: always
 
 .arm64-test-needs: &arm64-test-needs
   - alpine-3.18-arm64-rootfs-export
@@ -99,6 +109,7 @@
       - '*.dtb'
     when: always
   rules:
+    - !reference [.test-rules, pre_rules]
     - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
   tags:
     - xilinx
@@ -117,6 +128,7 @@
       - '*.log'
     when: always
   rules:
+    - !reference [.test-rules, pre_rules]
     - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
   tags:
     - xilinx
@@ -136,6 +148,7 @@
       - '*.log'
     when: always
   rules:
+    - !reference [.test-rules, pre_rules]
     - if: $QUBES_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
   tags:
     - qubes-hw2
-- 


Anthony Perard | Vates XCP-ng Developer

XCP-ng & Xen Orchestra - Vates solutions

web: https://vates.tech