Re: [XEN PATCH 3/3] CI: New stage "containers" to rebuild some containers

[Stefano Stabellini <sstabellini@xxxxxxxxxx>]

On Fri, 15 Nov 2024, Anthony PERARD wrote:
> Rebuild rolling release containers when XEN_CI_REBUILD_CONTAINERS is
> set. This is to be use with a scheduled pipeline.
> 
> When $XEN_CI_REBUILD_CONTAINERS is set, only build jobs related to the
> containers been rebuild will be executed.
> 
> Build jobs that are using one of the containers been rebuild should
> wait for the container to be rebuild. If it's a normal pipeline, those
> dependency are simply ignored.

This is a fantastic contribution, thanks Anthony!

I think we can simplify this patch by removing all stages except for
"containers" on the scheduled pipeline with XEN_CI_REBUILD_CONTAINERS
set to true.

I think it is a good idea to have a special schedule pipeline for this,
and we should exploit the fact that it is special and only use it to
rebuild the containers. If we want to, we can have a second scheduled
pipeline to do a full normal run afterwards.

This way, there is no need to carry the changes to build.yaml or
test.yaml that are a bit hard to read/understand for someone unfamiliar
with gitlab. When XEN_CI_REBUILD_CONTAINERS == true we only do the
containers stage.


> Signed-off-by: Anthony PERARD <anthony.perard@xxxxxxxxxx>
> ---
>  .gitlab-ci.yml                       |  2 +
>  automation/gitlab-ci/build.yaml      | 57 ++++++++++++++++++----------
>  automation/gitlab-ci/containers.yaml | 29 ++++++++++++++
>  automation/gitlab-ci/test.yaml       | 13 +++++++
>  4 files changed, 81 insertions(+), 20 deletions(-)
>  create mode 100644 automation/gitlab-ci/containers.yaml
> 
> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
> index 941e5822e8..dab9171086 100644
> --- a/.gitlab-ci.yml
> +++ b/.gitlab-ci.yml
> @@ -8,11 +8,13 @@ workflow:
>      - when: always
>  
>  stages:
> +  - containers
>    - analyze
>    - build
>    - test
>  
>  include:
> +  - 'automation/gitlab-ci/containers.yaml'
>    - 'automation/gitlab-ci/analyze.yaml'
>    - 'automation/gitlab-ci/build.yaml'
>    - 'automation/gitlab-ci/test.yaml'
> diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
> index 1ca6764225..e01eec1423 100644
> --- a/automation/gitlab-ci/build.yaml
> +++ b/automation/gitlab-ci/build.yaml
> @@ -3,6 +3,9 @@
>    image: ${XEN_REGISTRY}/${CONTAINER}
>    script:
>      - ./automation/scripts/build 2>&1 | tee build.log
> +  rules:
> +    - if: $XEN_CI_REBUILD_CONTAINERS && $XEN_CI_RUN_AFTER_CONTAINER_BUILD
> +    - if: $XEN_CI_REBUILD_CONTAINERS == null
>    artifacts:
>      paths:
>        - binaries/
> @@ -255,6 +258,8 @@
>  .test-jobs-artifact-common:
>    stage: build
>    needs: []
> +  rules:
> +    - if: $XEN_CI_REBUILD_CONTAINERS == null
>  
>  # Arm test artifacts
>  
> @@ -516,17 +521,24 @@ alpine-3.18-clang-debug:
>    variables:
>      CONTAINER: alpine:3.18
>  
> -archlinux-gcc:
> -  extends: .gcc-x86-64-build
> +.container-archlinux-current:
>    variables:
>      CONTAINER: archlinux:current
> +    XEN_CI_RUN_AFTER_CONTAINER_BUILD: true
> +  needs:
> +    - job: container-archlinux-current
> +      optional: true
>    allow_failure: true
>  
> +archlinux-gcc:
> +  extends:
> +    - .gcc-x86-64-build
> +    - .container-archlinux-current
> +
>  archlinux-gcc-debug:
> -  extends: .gcc-x86-64-build-debug
> -  variables:
> -    CONTAINER: archlinux:current
> -  allow_failure: true
> +  extends:
> +    - .gcc-x86-64-build-debug
> +    - .container-archlinux-current
>  
>  centos-7-gcc:
>    extends: .gcc-x86-64-build
> @@ -657,29 +669,34 @@ opensuse-leap-15.6-gcc-debug:
>    variables:
>      CONTAINER: opensuse:leap-15.6-x86_64
>  
> -opensuse-tumbleweed-clang:
> -  extends: .clang-x86-64-build
> +.container-opensuse-tumbleweed-x86_64:
>    variables:
>      CONTAINER: opensuse:tumbleweed-x86_64
> +    XEN_CI_RUN_AFTER_CONTAINER_BUILD: true
> +  needs:
> +    - job: container-opensuse-tumbleweed-x86_64
> +      optional: true
>    allow_failure: true
>  
> +opensuse-tumbleweed-clang:
> +  extends:
> +    - .clang-x86-64-build
> +    - .container-opensuse-tumbleweed-x86_64
> +
>  opensuse-tumbleweed-clang-debug:
> -  extends: .clang-x86-64-build-debug
> -  variables:
> -    CONTAINER: opensuse:tumbleweed-x86_64
> -  allow_failure: true
> +  extends:
> +    - .clang-x86-64-build-debug
> +    - .container-opensuse-tumbleweed-x86_64
>  
>  opensuse-tumbleweed-gcc:
> -  extends: .gcc-x86-64-build
> -  variables:
> -    CONTAINER: opensuse:tumbleweed-x86_64
> -  allow_failure: true
> +  extends:
> +    - .gcc-x86-64-build
> +    - .container-opensuse-tumbleweed-x86_64
>  
>  opensuse-tumbleweed-gcc-debug:
> -  extends: .gcc-x86-64-build-debug
> -  variables:
> -    CONTAINER: opensuse:tumbleweed-x86_64
> -  allow_failure: true
> +  extends:
> +    - .gcc-x86-64-build-debug
> +    - .container-opensuse-tumbleweed-x86_64
>  
>  # PowerPC builds (x86 cross)
>  debian-11-ppc64le-gcc:
> diff --git a/automation/gitlab-ci/containers.yaml 
> b/automation/gitlab-ci/containers.yaml
> new file mode 100644
> index 0000000000..25e8bdc34b
> --- /dev/null
> +++ b/automation/gitlab-ci/containers.yaml
> @@ -0,0 +1,29 @@
> +.container-build-tmpl:
> +  stage: containers
> +  image: docker:stable
> +  tags:
> +    - container-builder
> +  rules:
> +    - if: $XEN_CI_REBUILD_CONTAINERS
> +  services:
> +    - docker:dind
> +  before_script:
> +    - apk add make
> +    - docker info
> +    - docker login -u $CI_DEPLOY_USER -p $CI_DEPLOY_PASSWORD $CI_REGISTRY
> +  script:
> +    - make -C automation/build PUSH=1 REGISTRY=${XEN_REGISTRY} 
> ${CONTAINER/:/\/}
> +  after_script:
> +    - docker logout
> +
> +container-archlinux-current:
> +  extends:
> +    - .container-build-tmpl
> +  variables:
> +    CONTAINER: "archlinux:current"
> +
> +container-opensuse-tumbleweed-x86_64:
> +  extends:
> +    - .container-build-tmpl
> +  variables:
> +    CONTAINER: "opensuse:tumbleweed-x86_64"
> diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
> index 1822e3ea5f..e4d4f1f3c9 100644
> --- a/automation/gitlab-ci/test.yaml
> +++ b/automation/gitlab-ci/test.yaml
> @@ -1,6 +1,16 @@
> +.test-rules:
> +  # Only add rules which prevent jobs from been added to the pipeline. They
> +  # should all have "when: never".
> +  pre_rules:
> +    - if: $XEN_CI_REBUILD_CONTAINERS
> +      when: never
> +
>  .test-jobs-common:
>    stage: test
>    image: ${XEN_REGISTRY}/${CONTAINER}
> +  rules:
> +    - !reference [.test-rules, pre_rules]
> +    - when: always
>  
>  .arm64-test-needs: &arm64-test-needs
>    - alpine-3.18-arm64-rootfs-export
> @@ -99,6 +109,7 @@
>        - '*.dtb'
>      when: always
>    rules:
> +    - !reference [.test-rules, pre_rules]
>      - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
>    tags:
>      - xilinx
> @@ -117,6 +128,7 @@
>        - '*.log'
>      when: always
>    rules:
> +    - !reference [.test-rules, pre_rules]
>      - if: $XILINX_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
>    tags:
>      - xilinx
> @@ -136,6 +148,7 @@
>        - '*.log'
>      when: always
>    rules:
> +    - !reference [.test-rules, pre_rules]
>      - if: $QUBES_JOBS == "true" && $CI_COMMIT_REF_PROTECTED == "true"
>    tags:
>      - qubes-hw2
> -- 
> 
> 
> Anthony Perard | Vates XCP-ng Developer
> 
> XCP-ng & Xen Orchestra - Vates solutions
> 
> web: https://vates.tech
>