chore(gitlab-ci): introduce dynamic ci for image versioning; rename stages and jobs

2024-08-23 21:55:25 +02:00 · 2024-08-23 21:55:25 +02:00 · d2a80f20b6
commit d2a80f20b6
parent 6f5c74f1ca
3 changed files with 208 additions and 121 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -2,6 +2,19 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later

+### IMPORTANT NOTICE ###
+# Our pipeline consists of static and dynamic parts.
+# 
+# This file only contains the static parts of our pipeline.
+# Dynamic jobs are defined in .gitlab-ci/dynamic-ci.yml.
+# These are used as a template to generate jobs during the pipeline runtime.
+# 
+# The marker "#dyn#" (without quotes) will be replaced by concrete values.
+# 
+# TODO: documentation about dynamic stage
+###
+
+
 # workflow:
 #   rules:
 #     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@ -10,30 +23,12 @@
 #     - if: $CI_COMMIT_TAG =~ /^t/
 #     - if: $CI_COMMIT_TAG =~ /^d/

-variables:
-  # TODO: old vars; review and maybe remove
-  AWS_SHARED_CREDENTIALS_FILE: "/etc/aws/credentials"
-  TRANSFER_METER_FREQUENCY: "2s"
-  CONTAINER_REGISTRY: registry.uniworx.de/fradrive/fradrive
-
-
 default:
  image:
-    name: registry.uniworx.de/uniworx/containers/debian:12.5
+    name: ${CI_REGISTRY}/uniworx/containers/debian:12.5
    entrypoint: [""]
    docker:
      platform: x86_64
-  #   name: registry.uniworx.de/fradrive/fradrive/debian-podman:latest
-    # name: registry.uniworx.de/fradrive/fradrive/docker:dind
-  #   name: registry.uniworx.de/fradrive/fradrive/fradrive-env:latest # Debian 12.5 Bookworm
-  #   entrypoint: [""]
-  #   docker:
-  #     platform: x86_64
-  # #     user: ${ENV_USER}
-  # before_script:
-  #   - apt-get -y update
-  #   - apt-get -y install make podman
-  #   - chown -R ${ENV_USER} . # Change project directory ownership to (non-root) user inside docker container
  artifacts:
    name: "${CI_JOB_NAME}-${CI_COMMIT_SHORT_SHA}"
    expire_in: "1 day"
@ -42,115 +37,30 @@ default:


 stages:
-  - containers
-  - compile
-  - lint
-  - test
+  - dynamic
  - release # TODO

-# TODO: apply rules:changes for image jobs to only execute jobs when Dockerfiles change
-
-frontend image:
-  stage: containers
-  image: &buildah quay.io/buildah/stable:latest
+# TODO: documentation
+dynamic:prepare:
+  stage: dynamic
  script:
-    - buildah bud -t frontend docker/frontend/Dockerfile
-    - buildah push --creds "${CI_REGISTRY_USER}:${CI_JOB_TOKEN}" frontend $CONTAINER_REGISTRY/frontend
-  rules:
-    - changes:
-        - docker/frontend/Dockerfile
-
-
-backend image:
-  stage: containers
-  image: *buildah
-  script:
-    - buildah bud -t backend docker/backend/Dockerfile
-    - buildah push --creds "${CI_REGISTRY_USER}:${CI_JOB_TOKEN}" backend $CONTAINER_REGISTRY/backend
-  rules:
-    - changes:
-        - docker/backend/Dockerfile
-
-
-frontend build:
-  stage: compile
-  needs:
-    - job: frontend image
-      optional: true
-  script:
-    - &load-frontend-image zcat docker/frontend/image.tar.gz | podman image load
-    - make -- --docker-run-frontend-build FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose
+    - FRONTEND_IMAGE_VERSION=`git log docker/frontend/Dockerfile | grep '^commit' | wc --lines`
+    - BACKEND_IMAGE_VERSION=`git log docker/backend/Dockerfile | grep '^commit' | wc --lines`
+    - cat .gitlab-ci/dynamic-gitlab-ci.yml | .gitlab-ci/dynamci.pl FRONTEND_IMAGE_VERSION=${FRONTEND_IMAGE_VERSION} BACKEND_IMAGE_VERSION=${BACKEND_IMAGE_VERSION} > dynamic-gitlab-ci.yml
  artifacts:
    paths:
-      - /fradrive/node_modules
-      - /fradrive/well-known
-  cache:
-    - &frontend-cache
-      key: default-frontend
-      paths:
-        - /fradrive/.npm/
-        - /fradrive/.well-known-cache/
+      - dynamic-gitlab-ci.yml

-frontend lint:
-  stage: lint
+# TODO: documentation
+dynamic:execute:
+  stage: dynamic
  needs:
-    - job: frontend image
-      optional: true
-  script:
-    - *load-frontend-image
-    - make -- --docker-run-frontend-lint FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose
-  cache: *frontend-cache
-
-frontend test:
-  stage: test
-  needs:
-    - job: frontend image
-      optional: true
-    - job: frontend build
-      artifacts: true
-  script:
-    - *load-frontend-image
-    - make -- --docker-run-frontend-test FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose
-
-  cache: *frontend-cache
-
-backend build:
-  stage: compile
-  needs:
-    - job: backend image
-      optional: true
-    - job: frontend build
-      artifacts: true
-  script:
-    - &load-backend-image zcat docker/backend/image.tar.gz | podman image load
-    - make -- --docker-run-backend-build FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose
-  artifacts:
-    paths:
-      - /fradrive/.stack/
-      - /fradrive/.stack-work/
-  cache: &backend-cache
-
-backend lint:
-  stage: lint
-  needs:
-    - job: backend image
-      optional: true
-  script:
-    - *load-backend-image
-    - make -- --docker-run-backend-lint FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose
-  cache: *backend-cache
-
-backend test:
-  stage: test
-  needs:
-    - job: backend image
-      optional: true
-    - job: backend build
-      artifacts: true
-  script:
-    - *load-backend-image
-    - make -- --docker-run-backend-test FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose
-  cache: *backend-cache
+    - dynamic:prepare
+  trigger:
+    strategy: depend
+    include:
+      - artifact: dynamic-gitlab-ci.yml
+        job: dynamic:prepare

 # frontend dependencies:
 #   stage: setup
--- a/.gitlab-ci/dynamci.pl
+++ b/.gitlab-ci/dynamci.pl
@ -0,0 +1,43 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+my %kv = ();
+
+for(@ARGV) {
+  if(!m#^\s*([^=\s]+)\s*=\s*(.*?)\s*$#) {
+    die "$0: Bad parameter '$_': Not a key=value pair\n"
+  }
+  $kv{$1} = $2;
+}
+
+my $invar = 0;
+
+LOOP: while(my $line = <STDIN>) {
+  if(1==$invar) {
+    if($line=~m/^(\s+)(\S+)(\s*:\s*)(\S+)(.*)/) {
+      my ($pre1, $key, $pre2, $var, $post) = ($1, $2, $3, $4, $5);
+      if('#dyn#' eq $var) {
+        if(not exists $kv{$key}) {
+          die "$0: No value given for key '$key' in the parameters but it is defined in input!\n"
+        }
+        my $v = $kv{$key};
+        delete $kv{$key};
+        print "$pre1$key$pre2$v$post\n";
+        next LOOP;
+      }
+    } elsif($line=~m/^[^#\t ]/) {
+      $invar = 2
+    }
+  }
+  if(0==$invar and $line=~m#^\s*variables\s*:\s*$#) {
+    $invar = 1;
+  }
+  print $line;
+}
+
+my @rem = sort keys %kv;
+
+die "$0: Variables occur in parameter but not in input: @rem!\n" if @rem;
+
--- a/.gitlab-ci/dynamic-gitlab--ci.yml
+++ b/.gitlab-ci/dynamic-gitlab--ci.yml
@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: 2024 Sarah Vaupel <sarah.vaupel@uniworx.de>
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+### IMPORTANT NOTICE ###
+# Our pipeline consists of static and dynamic parts.
+# 
+# This file only contains the dynamic parts of our pipeline.
+# Static jobs are defined in .gitlab-ci.yml.
+# 
+# The marker "#dyn#" (without quotes) will be replaced by concrete values.
+###
+
+variables:
+  FRONTEND_IMAGE_VERSION: #dyn#
+  BACKEND_IMAGE_VERSION: #dyn#
+
+stages:
+  - containers
+  - compile
+  - lint
+  - test
+
+
+containers:frontend:
+  stage: containers
+  image: &buildah quay.io/buildah/stable:latest
+  script:
+    # - IMAGE_VERSION=`git log docker/frontend/Dockerfile | grep '^commit' | wc --lines`
+    - buildah bud -t frontend/${CI_COMMIT_REF_SLUG}:${FRONTEND_IMAGE_VERSION} docker/frontend/Dockerfile
+    - buildah push --creds "${CI_REGISTRY_USER}:${CI_JOB_TOKEN}" frontend/${CI_COMMIT_REF_SLUG}:${FRONTEND_IMAGE_VERSION} ${CI_REGISTRY_IMAGE}/frontend/${CI_COMMIT_REF_SLUG}:${FRONTEND_IMAGE_VERSION}
+  rules:
+    - changes:
+        - docker/frontend/Dockerfile
+  interruptible: false
+
+containers:backend:
+  stage: containers
+  image: *buildah
+  script:
+    # - IMAGE_VERSION=`git log docker/backend/Dockerfile | grep '^commit' | wc --lines`
+    - buildah bud -t backend/${CI_COMMIT_REF_SLUG}:${BACKEND_IMAGE_VERSION} docker/backend/Dockerfile
+    - buildah push --creds "${CI_REGISTRY_USER}:${CI_JOB_TOKEN}" backend/${CI_COMMIT_REF_SLUG}:${BACKEND_IMAGE_VERSION} ${CI_REGISTRY_IMAGE}/backend/${CI_COMMIT_REF_SLUG}:${BACKEND_IMAGE_VERSION}
+  rules:
+    - changes:
+        - docker/backend/Dockerfile
+  interruptible: false
+
+
+frontend build:
+  stage: compile
+  needs:
+    - job: containers:frontend
+      optional: true
+  image: &frontend-image ${CI_REGISTRY_IMAGE}/frontend/${CI_COMMIT_REF_SLUG}:${FRONTEND_IMAGE_VERSION}
+  script:
+    - &load-frontend-image zcat docker/frontend/image.tar.gz | podman image load
+    - make -- --docker-run-frontend-build FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose
+  artifacts:
+    paths:
+      - /fradrive/node_modules
+      - /fradrive/well-known
+  cache:
+    - &frontend-cache
+      key: default-frontend
+      paths:
+        - /fradrive/.npm/
+        - /fradrive/.well-known-cache/
+
+frontend lint:
+  stage: lint
+  needs:
+    - job: containers:frontend
+      optional: true
+  image: *frontend-image
+  script:
+    - *load-frontend-image
+    - make -- --docker-run-frontend-lint FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose
+  cache: *frontend-cache
+
+frontend test:
+  stage: test
+  needs:
+    - job: containers:frontend
+      optional: true
+    - job: frontend build
+      artifacts: true
+  image: *frontend-image
+  script:
+    - *load-frontend-image
+    - make -- --docker-run-frontend-test FRADRIVE_SERVICE=frontend CONTAINER_RUNNER=podman-compose
+
+  cache: *frontend-cache
+
+backend build:
+  stage: compile
+  needs:
+    - job: containers:backend
+      optional: true
+    - job: frontend build
+      artifacts: true
+  image: &backend-image ${CI_REGISTRY_IMAGE}/backend/${CI_COMMIT_REF_SLUG}:${BACKEND_IMAGE_VERSION}
+  script:
+    - &load-backend-image zcat docker/backend/image.tar.gz | podman image load
+    - make -- --docker-run-backend-build FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose
+  artifacts:
+    paths:
+      - /fradrive/.stack/
+      - /fradrive/.stack-work/
+  cache: &backend-cache
+
+backend lint:
+  stage: lint
+  needs:
+    - job: containers:backend
+      optional: true
+  image: *backend-image
+  script:
+    - *load-backend-image
+    - make -- --docker-run-backend-lint FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose
+  cache: *backend-cache
+
+backend test:
+  stage: test
+  needs:
+    - job: containers:backend
+      optional: true
+    - job: backend build
+      artifacts: true
+  image: *backend-image
+  script:
+    - *load-backend-image
+    - make -- --docker-run-backend-test FRADRIVE_SERVICE=backend CONTAINER_RUNNER=podman-compose
+  cache: *backend-cache