fix
This commit is contained in:
hcornet committed
1 parent
a4f5cb5bb8
commit
4d6db44320
7 files changed
+809
-117
No files matched your search
@@ -0,0 +1,12 @@
|
||||
# /opt/openbao/.env
|
||||
|
||||
# Domaine racine (routage Traefik + tunnel Cloudflare)
|
||||
DOMAIN=tips-of-mine.com
|
||||
|
||||
# Version d'OpenBao — à figer explicitement, jamais de :latest sur un coffre.
|
||||
# Vérifier le tag exact disponible avant de démarrer :
|
||||
# git ls-remote --tags --refs https://github.com/openbao/openbao | tail -5
|
||||
OPENBAO_VERSION=2.6.1
|
||||
|
||||
OPENBAO_UID=100
|
||||
OPENBAO_GID=1000
|
||||
@@ -1,66 +0,0 @@
|
||||
# Template
|
||||
name: Deployment Verification
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- develop
|
||||
tags:
|
||||
- "v[0-9]+.[0-9]+.[0-9]"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# https://github.com/docker/setup-qemu-action#usage
|
||||
- name: Set up QEMU
|
||||
uses: docker/[email protected]
|
||||
|
||||
# https://github.com/marketplace/actions/docker-setup-buildx
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/[email protected]
|
||||
|
||||
#
|
||||
- name: Create necessary Docker networks
|
||||
run: |
|
||||
docker network create back_network || true
|
||||
docker network create traefik_front_network || true
|
||||
|
||||
- name: Start up services using Docker Compose
|
||||
run: docker compose -f docker-compose.yml up -d
|
||||
|
||||
# - name: Modify /etc/hosts for internal routing
|
||||
# run: |
|
||||
# echo "127.0.0.1 gitea.tips-of-mine.com" | sudo tee -a /etc/hosts
|
||||
|
||||
- name: Print Docker Compose services status
|
||||
run: docker ps
|
||||
|
||||
- name: Wait for the application to be ready via Traefik
|
||||
run: |
|
||||
echo "Checking the routing and availability of application via Traefik..."
|
||||
timeout 5m bash -c 'while ! curl -fsSLk "https://openbao.tips-of-mine.com"; do echo "Waiting for the application to be ready..."; sleep 10; done'
|
||||
|
||||
- name: Inspect Network Configuration
|
||||
run: |
|
||||
docker network inspect back_network
|
||||
docker network inspect traefik_front_network
|
||||
|
||||
- name: Shutdown Docker Compose services
|
||||
if: always()
|
||||
run: docker compose -f docker-compose.yml down
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
docker compose --profile setup down
|
||||
rm -rf /workspace/tips-of-mine/openbao/*
|
||||
@@ -1,51 +0,0 @@
|
||||
# Template
|
||||
name: Deployment Verification
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
deploy-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Create necessary Docker networks
|
||||
run: |
|
||||
docker network create back_network_{ vars.APPLICATION_NAME } || true
|
||||
docker network create traefik_front_network || true
|
||||
|
||||
- name: Start up services using Docker Compose
|
||||
run: docker compose -f docker-compose.yml up -d
|
||||
|
||||
# - name: Modify /etc/hosts for internal routing
|
||||
# run: |
|
||||
# echo "127.0.0.1 { vars.APPLICATION_URL }" | sudo tee -a /etc/hosts
|
||||
# echo "127.0.0.1 dashboard.tips-of-mine.com" | sudo tee -a /etc/hosts
|
||||
|
||||
- name: Print Docker Compose services status
|
||||
run: docker ps
|
||||
|
||||
- name: Wait for the application to be ready via Traefik
|
||||
run: |
|
||||
echo "Checking the routing and availability of application via Traefik..."
|
||||
timeout 5m bash -c 'while ! curl -fsSLk "https://{ vars.APPLICATION_URL }"; do echo "Waiting for the application to be ready..."; sleep 10; done'
|
||||
|
||||
- name: Inspect Network Configuration
|
||||
run: |
|
||||
docker network inspect back_network_{ vars.APPLICATION_NAME }
|
||||
docker network inspect traefik_front_network
|
||||
|
||||
- name: Shutdown Docker Compose services
|
||||
if: always()
|
||||
run: docker compose -f docker-compose.yml down
|
||||
@@ -0,0 +1,73 @@
|
||||
# Template — vérification isolée d'une stack Docker Compose.
|
||||
# Remplace ci-develop.yml et ci-main.yml : les supprimer du dépôt.
|
||||
# Logique commune : .gitea/scripts/compose-ci.sh et compose-ci-rewrite.py
|
||||
name: Compose CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- develop
|
||||
- main
|
||||
tags:
|
||||
- "v[0-9]+.[0-9]+.[0-9]+"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
CI_SCRIPT: .gitea/scripts/compose-ci.sh
|
||||
HELPER_IMAGE: python:3.13-alpine3.22
|
||||
SHELLCHECK_IMAGE: koalaman/shellcheck:v0.10.0
|
||||
|
||||
jobs:
|
||||
lint-ci-scripts:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: ShellCheck du script de CI
|
||||
run: docker run --rm -i --network none "$SHELLCHECK_IMAGE" - < "$CI_SCRIPT"
|
||||
|
||||
- name: Syntaxe du script de réécriture
|
||||
run: >-
|
||||
docker run --rm -i --network none "$HELPER_IMAGE"
|
||||
python3 -c "import ast, sys; ast.parse(sys.stdin.read())"
|
||||
< .gitea/scripts/compose-ci-rewrite.py
|
||||
|
||||
compose-verify:
|
||||
needs: lint-ci-scripts
|
||||
if: vars.CI_SKIP_COMPOSE != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
CI_COMPOSE_FILE: ${{ vars.CI_COMPOSE_FILE }}
|
||||
CI_MEM_LIMIT: ${{ vars.CI_MEM_LIMIT }}
|
||||
CI_CPU_LIMIT: ${{ vars.CI_CPU_LIMIT }}
|
||||
CI_PIDS_LIMIT: ${{ vars.CI_PIDS_LIMIT }}
|
||||
CI_MIN_AVAILABLE_MB: ${{ vars.CI_MIN_AVAILABLE_MB }}
|
||||
CI_WAIT_TIMEOUT: ${{ vars.CI_WAIT_TIMEOUT }}
|
||||
CI_LOG_LINES: ${{ vars.CI_LOG_LINES }}
|
||||
CI_DRY_RUN: ${{ vars.CI_DRY_RUN }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Préparer une configuration isolée
|
||||
run: bash "$CI_SCRIPT" prepare
|
||||
|
||||
- name: Démarrer la stack de CI
|
||||
run: bash "$CI_SCRIPT" up
|
||||
|
||||
- name: Vérifier l'état des conteneurs
|
||||
run: bash "$CI_SCRIPT" verify
|
||||
|
||||
- name: Journaux (en cas d'échec)
|
||||
if: failure()
|
||||
run: bash "$CI_SCRIPT" logs
|
||||
|
||||
- name: Nettoyer la stack de CI
|
||||
if: always()
|
||||
run: bash "$CI_SCRIPT" cleanup
|
||||
@@ -12,3 +12,7 @@
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
# Variables d'environnement et secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
@@ -0,0 +1,398 @@
|
||||
"""Réécrit une configuration Compose (sortie de `docker compose config --format json`)
|
||||
pour qu'elle puisse tourner en CI sur le même démon Docker que la production,
|
||||
sans jamais la toucher.
|
||||
|
||||
Exécuté dans un conteneur utilitaire lancé par compose-ci.sh :
|
||||
- stdin : configuration Compose résolue (JSON, contient des secrets : jamais affichée)
|
||||
- stdout : ligne 1 = nombre de services retenus
|
||||
ligne 2 = configuration réécrite (JSON sur une ligne)
|
||||
lignes suivantes = valeurs sensibles à masquer dans les journaux (base64)
|
||||
- stderr : rapport lisible (aucune valeur sensible)
|
||||
- /ci : copie du dépôt sur l'hôte (répertoire CI_HOST_DIR vu depuis ce conteneur)
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import sys
|
||||
|
||||
LOCAL_DIR = "/ci"
|
||||
|
||||
PROJECT_RE = re.compile(r"^ci-[a-z0-9]([a-z0-9-]*[a-z0-9])?-[0-9]+-[0-9]+$")
|
||||
MEM_RE = re.compile(r"^[0-9]+[bkmg]?$")
|
||||
|
||||
# Labels lus par des composants de production (routage, découverte, mises à jour).
|
||||
DROPPED_LABEL_PREFIXES = (
|
||||
"traefik.",
|
||||
"homepage.",
|
||||
"com.centurylinklabs.watchtower.",
|
||||
"sablier.",
|
||||
)
|
||||
CI_LABELS = {
|
||||
"traefik.enable": "false",
|
||||
"com.centurylinklabs.watchtower.enable": "false",
|
||||
"tips-of-mine.ci": "true",
|
||||
}
|
||||
|
||||
# Capacités non isolées par les namespaces : elles agissent sur l'hôte.
|
||||
DANGEROUS_CAPS = {
|
||||
"ALL", "SYS_ADMIN", "SYS_MODULE", "SYS_RAWIO", "SYS_TIME", "SYS_BOOT",
|
||||
"SYSLOG", "DAC_READ_SEARCH", "MAC_ADMIN", "MAC_OVERRIDE", "BPF", "PERFMON",
|
||||
}
|
||||
HOST_NAMESPACE_KEYS = ("pid", "ipc", "uts", "userns_mode", "cgroup")
|
||||
|
||||
# Répertoires hôtes qui contiennent (ou sont parents de) sockets de démons.
|
||||
SOCKET_PARENTS = {"/", "/run", "/var", "/var/run"}
|
||||
|
||||
SENSITIVE_NAME_RE = re.compile(
|
||||
r"PASS|PWD|SECRET|TOKEN|KEY|CREDENTIAL|PRIVATE|AUTH|DSN|URI|URL|SALT|COOKIE",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
MASK_MIN_LENGTH = 6
|
||||
OOM_SCORE_ADJ = 500
|
||||
|
||||
|
||||
def fail(message):
|
||||
print(f"ERREUR : {message}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def report(message):
|
||||
print(message, file=sys.stderr)
|
||||
|
||||
|
||||
def read_settings():
|
||||
project = os.environ.get("CI_PROJECT", "")
|
||||
workspace = os.environ.get("CI_WORKSPACE", "")
|
||||
host_dir = os.environ.get("CI_HOST_DIR", "")
|
||||
mem = (os.environ.get("CI_MEM_LIMIT") or "1g").lower()
|
||||
cpus = os.environ.get("CI_CPU_LIMIT") or "1.0"
|
||||
pids = os.environ.get("CI_PIDS_LIMIT") or "2048"
|
||||
|
||||
if not PROJECT_RE.match(project):
|
||||
fail(f"nom de projet refusé : {project!r}")
|
||||
if not workspace.startswith("/"):
|
||||
fail("CI_WORKSPACE doit être un chemin absolu")
|
||||
if host_dir != f"/var/tmp/gitea-ci/{project}":
|
||||
fail(f"répertoire hôte refusé : {host_dir!r}")
|
||||
if not MEM_RE.match(mem):
|
||||
fail(f"CI_MEM_LIMIT invalide : {mem!r}")
|
||||
try:
|
||||
cpus_value = float(cpus)
|
||||
pids_value = int(pids)
|
||||
except ValueError:
|
||||
fail("CI_CPU_LIMIT ou CI_PIDS_LIMIT invalide")
|
||||
if cpus_value <= 0 or pids_value <= 0:
|
||||
fail("CI_CPU_LIMIT et CI_PIDS_LIMIT doivent être positifs")
|
||||
|
||||
return {
|
||||
"project": project,
|
||||
"workspace": posixpath.normpath(workspace),
|
||||
"host_dir": host_dir,
|
||||
"mem": mem,
|
||||
"cpus": cpus_value,
|
||||
"pids": pids_value,
|
||||
}
|
||||
|
||||
|
||||
def is_under(path, root):
|
||||
path = posixpath.normpath(path)
|
||||
return path == root or path.startswith(root + "/")
|
||||
|
||||
|
||||
def relative_to_workspace(path, settings):
|
||||
return posixpath.relpath(posixpath.normpath(path), settings["workspace"])
|
||||
|
||||
|
||||
def host_path_problem(path, read_only):
|
||||
"""Raison d'exclusion d'un montage hôte situé hors du dépôt, ou None."""
|
||||
path = posixpath.normpath(path)
|
||||
if path.endswith(".sock") or "docker.sock" in path:
|
||||
return "socket de l'hôte"
|
||||
if path in SOCKET_PARENTS or path.startswith(("/run/", "/var/run/")):
|
||||
return "répertoire de sockets de l'hôte"
|
||||
if path == "/dev" or path.startswith("/dev/"):
|
||||
return "périphérique de l'hôte"
|
||||
if not read_only:
|
||||
return "montage de l'hôte en écriture"
|
||||
return None
|
||||
|
||||
|
||||
def copied_path_problem(path, settings):
|
||||
"""Refuse un chemin du dépôt qui sort de la copie via un lien symbolique."""
|
||||
rel = relative_to_workspace(path, settings)
|
||||
local = posixpath.normpath(posixpath.join(LOCAL_DIR, rel))
|
||||
if not is_under(os.path.realpath(local), LOCAL_DIR):
|
||||
return f"lien symbolique sortant du dépôt ({rel})"
|
||||
return None
|
||||
|
||||
|
||||
def file_resource_problem(resource, settings):
|
||||
"""Contrôle un secret ou une config de premier niveau déclaré par fichier."""
|
||||
source = (resource or {}).get("file")
|
||||
if not source:
|
||||
return None
|
||||
if is_under(source, settings["workspace"]):
|
||||
return copied_path_problem(source, settings)
|
||||
return host_path_problem(source, read_only=True)
|
||||
|
||||
|
||||
def service_problems(svc, settings, unsafe_secrets, unsafe_configs):
|
||||
reasons = []
|
||||
if svc.get("privileged"):
|
||||
reasons.append("privileged")
|
||||
for key in HOST_NAMESPACE_KEYS:
|
||||
if svc.get(key) == "host":
|
||||
reasons.append(f"{key}: host")
|
||||
network_mode = svc.get("network_mode") or ""
|
||||
if network_mode == "host" or network_mode.startswith("container:"):
|
||||
reasons.append(f"network_mode: {network_mode}")
|
||||
|
||||
reservations = ((svc.get("deploy") or {}).get("resources") or {}).get("reservations") or {}
|
||||
if svc.get("devices") or svc.get("device_cgroup_rules") or svc.get("gpus") or reservations.get("devices"):
|
||||
reasons.append("accès à des périphériques")
|
||||
|
||||
caps = {str(cap).upper().removeprefix("CAP_") for cap in svc.get("cap_add") or []}
|
||||
if caps & DANGEROUS_CAPS:
|
||||
reasons.append("cap_add " + ",".join(sorted(caps & DANGEROUS_CAPS)))
|
||||
|
||||
for entry in svc.get("volumes_from") or []:
|
||||
if str(entry).startswith("container:"):
|
||||
reasons.append("volumes_from vers un conteneur externe")
|
||||
|
||||
for volume in svc.get("volumes") or []:
|
||||
if volume.get("type") != "bind":
|
||||
continue
|
||||
source = volume.get("source") or ""
|
||||
if is_under(source, settings["workspace"]):
|
||||
problem = copied_path_problem(source, settings)
|
||||
else:
|
||||
problem = host_path_problem(source, bool(volume.get("read_only")))
|
||||
if problem:
|
||||
problem = f"{problem} ({source})"
|
||||
if problem:
|
||||
reasons.append(problem)
|
||||
|
||||
for ref in svc.get("secrets") or []:
|
||||
if ref.get("source") in unsafe_secrets:
|
||||
reasons.append(f"secret {ref.get('source')} non isolable")
|
||||
for ref in svc.get("configs") or []:
|
||||
if ref.get("source") in unsafe_configs:
|
||||
reasons.append(f"config {ref.get('source')} non isolable")
|
||||
return reasons
|
||||
|
||||
|
||||
def dependency_targets(svc):
|
||||
"""Services dont celui-ci ne peut pas se passer (réseau ou volumes partagés)."""
|
||||
targets = []
|
||||
network_mode = svc.get("network_mode") or ""
|
||||
if network_mode.startswith("service:"):
|
||||
targets.append(network_mode.removeprefix("service:"))
|
||||
for entry in svc.get("volumes_from") or []:
|
||||
entry = str(entry)
|
||||
if not entry.startswith("container:"):
|
||||
targets.append(entry.removeprefix("service:").split(":")[0])
|
||||
return targets
|
||||
|
||||
|
||||
def select_services(cfg, settings):
|
||||
services = cfg.get("services") or {}
|
||||
unsafe_secrets = {
|
||||
name for name, res in (cfg.get("secrets") or {}).items()
|
||||
if file_resource_problem(res, settings)
|
||||
}
|
||||
unsafe_configs = {
|
||||
name for name, res in (cfg.get("configs") or {}).items()
|
||||
if file_resource_problem(res, settings)
|
||||
}
|
||||
|
||||
excluded = {}
|
||||
for name, svc in services.items():
|
||||
reasons = service_problems(svc, settings, unsafe_secrets, unsafe_configs)
|
||||
if reasons:
|
||||
excluded[name] = reasons
|
||||
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for name, svc in services.items():
|
||||
if name in excluded:
|
||||
continue
|
||||
missing = sorted({t for t in dependency_targets(svc) if t in excluded})
|
||||
if missing:
|
||||
excluded[name] = [f"dépend du service exclu {', '.join(missing)}"]
|
||||
changed = True
|
||||
return excluded
|
||||
|
||||
|
||||
def collect_masks(services):
|
||||
masks = set()
|
||||
for svc in services.values():
|
||||
environment = svc.get("environment") or {}
|
||||
if isinstance(environment, list):
|
||||
environment = dict(item.split("=", 1) for item in environment if "=" in item)
|
||||
for key, value in environment.items():
|
||||
if value is None or not SENSITIVE_NAME_RE.search(key):
|
||||
continue
|
||||
value = str(value)
|
||||
# `compose config` échappe les $ en $$ ; le conteneur voit la forme simple.
|
||||
for variant in {value, value.replace("$$", "$")}:
|
||||
if len(variant) >= MASK_MIN_LENGTH:
|
||||
masks.add(variant)
|
||||
return masks
|
||||
|
||||
|
||||
def relocate_source(source, settings, create_missing):
|
||||
rel = relative_to_workspace(source, settings)
|
||||
local = posixpath.normpath(posixpath.join(LOCAL_DIR, rel))
|
||||
if create_missing and not os.path.lexists(local):
|
||||
os.makedirs(local, exist_ok=True)
|
||||
os.chmod(local, 0o777)
|
||||
if rel == ".":
|
||||
return settings["host_dir"]
|
||||
return posixpath.join(settings["host_dir"], rel)
|
||||
|
||||
|
||||
def rewrite_labels(labels, project):
|
||||
if isinstance(labels, list):
|
||||
labels = dict(item.split("=", 1) if "=" in item else (item, "") for item in labels)
|
||||
kept = {k: v for k, v in (labels or {}).items() if not k.startswith(DROPPED_LABEL_PREFIXES)}
|
||||
kept.update(CI_LABELS)
|
||||
kept["tips-of-mine.ci.project"] = project
|
||||
return kept
|
||||
|
||||
|
||||
def apply_limits(svc, settings):
|
||||
deploy = svc.get("deploy") or {}
|
||||
deploy.pop("restart_policy", None)
|
||||
limits = (deploy.get("resources") or {}).get("limits")
|
||||
|
||||
if limits is not None:
|
||||
# Compose refuse deux valeurs distinctes (ex. cpus et deploy...limits.cpus) :
|
||||
# on complète le bloc déjà déclaré par le service.
|
||||
if not svc.get("mem_limit") and not limits.get("memory"):
|
||||
limits["memory"] = settings["mem"]
|
||||
if not svc.get("cpus") and not limits.get("cpus"):
|
||||
limits["cpus"] = str(settings["cpus"])
|
||||
if not svc.get("pids_limit") and not limits.get("pids"):
|
||||
limits["pids"] = settings["pids"]
|
||||
else:
|
||||
svc.setdefault("mem_limit", settings["mem"])
|
||||
svc.setdefault("cpus", settings["cpus"])
|
||||
svc.setdefault("pids_limit", settings["pids"])
|
||||
|
||||
# Pas de swap pour la CI : le swap de l'hôte sert à la production.
|
||||
memory = svc.get("mem_limit") or (limits or {}).get("memory")
|
||||
if memory and not svc.get("memswap_limit"):
|
||||
svc["memswap_limit"] = memory
|
||||
svc["oom_score_adj"] = max(int(svc.get("oom_score_adj") or 0), OOM_SCORE_ADJ)
|
||||
|
||||
if deploy:
|
||||
svc["deploy"] = deploy
|
||||
else:
|
||||
svc.pop("deploy", None)
|
||||
|
||||
|
||||
def rewrite_service(name, svc, excluded, settings):
|
||||
for key in ("container_name", "ports", "logging", "mac_address", "external_links"):
|
||||
svc.pop(key, None)
|
||||
if "build" in svc:
|
||||
# Image nommée <projet>-<service> : aucun tag de production n'est écrasé.
|
||||
svc.pop("image", None)
|
||||
elif svc.get("pull_policy") not in (None, "missing", "if_not_present", "never"):
|
||||
svc["pull_policy"] = "missing"
|
||||
svc["restart"] = "no"
|
||||
svc["labels"] = rewrite_labels(svc.get("labels"), settings["project"])
|
||||
apply_limits(svc, settings)
|
||||
|
||||
networks = svc.get("networks")
|
||||
if isinstance(networks, dict):
|
||||
for attachment in networks.values():
|
||||
if isinstance(attachment, dict):
|
||||
for key in ("ipv4_address", "ipv6_address", "link_local_ips", "mac_address"):
|
||||
attachment.pop(key, None)
|
||||
|
||||
depends_on = svc.get("depends_on")
|
||||
if isinstance(depends_on, dict):
|
||||
for target in [t for t in depends_on if t in excluded]:
|
||||
report(f" - {name} : dépendance vers {target} retirée (service exclu)")
|
||||
del depends_on[target]
|
||||
if not depends_on:
|
||||
svc.pop("depends_on")
|
||||
if svc.get("links"):
|
||||
svc["links"] = [l for l in svc["links"] if str(l).split(":")[0] not in excluded]
|
||||
if not svc["links"]:
|
||||
svc.pop("links")
|
||||
|
||||
relocated = 0
|
||||
for volume in svc.get("volumes") or []:
|
||||
if volume.get("type") == "bind" and is_under(volume.get("source") or "", settings["workspace"]):
|
||||
volume["source"] = relocate_source(volume["source"], settings, create_missing=True)
|
||||
relocated += 1
|
||||
return relocated
|
||||
|
||||
|
||||
def rewrite_top_level(cfg, settings):
|
||||
project = settings["project"]
|
||||
for key, network in list((cfg.get("networks") or {}).items()):
|
||||
network = network or {}
|
||||
for field in ("external", "driver", "driver_opts", "ipam", "enable_ipv6"):
|
||||
network.pop(field, None)
|
||||
network["name"] = f"{project}_{key}"
|
||||
cfg["networks"][key] = network
|
||||
|
||||
for key, volume in list((cfg.get("volumes") or {}).items()):
|
||||
volume = volume or {}
|
||||
for field in ("external", "driver", "driver_opts"):
|
||||
volume.pop(field, None)
|
||||
volume["name"] = f"{project}_{key}"
|
||||
cfg["volumes"][key] = volume
|
||||
|
||||
for section in ("secrets", "configs"):
|
||||
for resource in (cfg.get(section) or {}).values():
|
||||
source = (resource or {}).get("file")
|
||||
if source and is_under(source, settings["workspace"]):
|
||||
resource["file"] = relocate_source(source, settings, create_missing=False)
|
||||
|
||||
cfg["name"] = project
|
||||
|
||||
|
||||
def main():
|
||||
settings = read_settings()
|
||||
try:
|
||||
cfg = json.load(sys.stdin)
|
||||
except json.JSONDecodeError:
|
||||
fail("configuration Compose illisible")
|
||||
|
||||
services = cfg.get("services") or {}
|
||||
if not services:
|
||||
fail("aucun service dans la configuration Compose")
|
||||
|
||||
masks = collect_masks(services)
|
||||
excluded = select_services(cfg, settings)
|
||||
for name in excluded:
|
||||
del services[name]
|
||||
|
||||
report(f"Projet de CI : {settings['project']}")
|
||||
report(f"Services testés : {', '.join(sorted(services)) or 'aucun'}")
|
||||
if excluded:
|
||||
report("Services exclus :")
|
||||
for name in sorted(excluded):
|
||||
report(f" - {name} : {'; '.join(excluded[name])}")
|
||||
|
||||
relocated = sum(rewrite_service(n, s, excluded, settings) for n, s in services.items())
|
||||
rewrite_top_level(cfg, settings)
|
||||
cfg["services"] = services
|
||||
report(f"Montages relocalisés : {relocated}")
|
||||
|
||||
out = sys.stdout
|
||||
out.write(f"{len(services)}\n")
|
||||
out.write(json.dumps(cfg, separators=(",", ":")) + "\n")
|
||||
for value in sorted(masks):
|
||||
out.write(base64.b64encode(value.encode()).decode() + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env bash
|
||||
# Vérification d'une stack Docker Compose en CI, sur le démon Docker de l'hôte,
|
||||
# sans aucune interaction avec les conteneurs, réseaux et volumes de production.
|
||||
#
|
||||
# Usage : compose-ci.sh prepare|up|verify|logs|cleanup
|
||||
#
|
||||
# Variables facultatives (variables Gitea de dépôt ou d'organisation) :
|
||||
# CI_COMPOSE_FILE fichier Compose à tester (défaut : détection automatique)
|
||||
# CI_MEM_LIMIT mémoire par service sans limite propre (défaut : 1g)
|
||||
# CI_CPU_LIMIT CPU par service sans limite propre (défaut : 1.0)
|
||||
# CI_PIDS_LIMIT processus par service sans limite propre (défaut : 2048)
|
||||
# CI_MIN_AVAILABLE_MB mémoire disponible exigée sur l'hôte (défaut : 3072)
|
||||
# CI_WAIT_TIMEOUT délai de démarrage en secondes (défaut : 300)
|
||||
# CI_LOG_LINES lignes de journal par service en cas d'échec (défaut : 50)
|
||||
# CI_DRY_RUN true : prépare et affiche le rapport sans rien démarrer
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
readonly HOST_ROOT="/var/tmp/gitea-ci"
|
||||
readonly HELPER_IMAGE="python:3.13-alpine3.22"
|
||||
readonly CI_MARKER_LABEL="tips-of-mine.ci"
|
||||
readonly STABILIZATION_SECONDS=15
|
||||
readonly POLL_SECONDS=5
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
readonly SCRIPT_DIR
|
||||
readonly WORK_DIR="${RUNNER_TEMP:-/tmp}/compose-ci"
|
||||
readonly CI_FILE="${WORK_DIR}/compose.ci.json"
|
||||
readonly COUNT_FILE="${WORK_DIR}/services.count"
|
||||
readonly MASK_FILE="${WORK_DIR}/mask.b64"
|
||||
|
||||
PROJECT=""
|
||||
SERVICE_COUNT=0
|
||||
|
||||
die() {
|
||||
echo "ERREUR : $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
is_dry_run() {
|
||||
[[ "${CI_DRY_RUN:-}" == "true" ]]
|
||||
}
|
||||
|
||||
project_name() {
|
||||
local repo run_id attempt name
|
||||
repo="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY absent}"
|
||||
repo="${repo##*/}"
|
||||
repo="$(printf '%s' "${repo,,}" | tr -c 'a-z0-9' '-' | tr -s '-')"
|
||||
repo="${repo#-}"
|
||||
repo="${repo:0:40}"
|
||||
repo="${repo%-}"
|
||||
run_id="${GITHUB_RUN_ID:?GITHUB_RUN_ID absent}"
|
||||
attempt="${GITHUB_RUN_ATTEMPT:-1}"
|
||||
name="ci-${repo}-${run_id}-${attempt}"
|
||||
[[ "$name" =~ ^ci-[a-z0-9]([a-z0-9-]*[a-z0-9])?-[0-9]+-[0-9]+$ ]] \
|
||||
|| die "nom de projet de CI invalide : ${name}"
|
||||
printf '%s' "$name"
|
||||
}
|
||||
|
||||
compose_file() {
|
||||
local candidate
|
||||
if [[ -n "${CI_COMPOSE_FILE:-}" ]]; then
|
||||
[[ "$CI_COMPOSE_FILE" != /* && "$CI_COMPOSE_FILE" != *..* ]] \
|
||||
|| die "CI_COMPOSE_FILE doit être un chemin relatif au dépôt"
|
||||
[[ -f "${GITHUB_WORKSPACE}/${CI_COMPOSE_FILE}" ]] \
|
||||
|| die "fichier Compose introuvable : ${CI_COMPOSE_FILE}"
|
||||
printf '%s' "$CI_COMPOSE_FILE"
|
||||
return
|
||||
fi
|
||||
for candidate in docker-compose.yml docker-compose.yaml compose.yaml compose.yml; do
|
||||
if [[ -f "${GITHUB_WORKSPACE}/${candidate}" ]]; then
|
||||
printf '%s' "$candidate"
|
||||
return
|
||||
fi
|
||||
done
|
||||
die "aucun fichier Compose trouvé à la racine du dépôt"
|
||||
}
|
||||
|
||||
compose() {
|
||||
docker compose --ansi never \
|
||||
--project-name "$PROJECT" \
|
||||
--project-directory "$GITHUB_WORKSPACE" \
|
||||
-f "$CI_FILE" "$@"
|
||||
}
|
||||
|
||||
helper() {
|
||||
docker run --rm --network none "$@"
|
||||
}
|
||||
|
||||
load_state() {
|
||||
PROJECT="$(project_name)"
|
||||
[[ -f "$CI_FILE" && -f "$COUNT_FILE" ]] \
|
||||
|| die "configuration de CI absente : l'étape de préparation n'a pas abouti"
|
||||
SERVICE_COUNT="$(<"$COUNT_FILE")"
|
||||
[[ "$SERVICE_COUNT" =~ ^[0-9]+$ ]] || die "nombre de services illisible"
|
||||
}
|
||||
|
||||
check_memory() {
|
||||
local min="${CI_MIN_AVAILABLE_MB:-3072}" available
|
||||
[[ "$min" =~ ^[0-9]+$ ]] || die "CI_MIN_AVAILABLE_MB invalide"
|
||||
available="$(awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo)"
|
||||
[[ "$available" =~ ^[0-9]+$ ]] || die "mémoire disponible illisible"
|
||||
echo "Mémoire disponible sur l'hôte : ${available} Mio (seuil : ${min} Mio)"
|
||||
(( available >= min )) \
|
||||
|| die "mémoire insuffisante sur l'hôte : démarrage annulé pour protéger la production"
|
||||
}
|
||||
|
||||
assert_isolated() {
|
||||
local key
|
||||
for key in '"container_name":' '"ports":' '"external":'; do
|
||||
if grep -qF "$key" "$CI_FILE"; then
|
||||
die "la configuration de CI contient encore ${key} : arrêt par sécurité"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
cmd_prepare() {
|
||||
local file raw
|
||||
PROJECT="$(project_name)"
|
||||
file="$(compose_file)"
|
||||
echo "Fichier Compose : ${file}"
|
||||
check_memory
|
||||
|
||||
install -d -m 0700 "$WORK_DIR"
|
||||
|
||||
echo "Copie du dépôt vers ${HOST_ROOT}/${PROJECT} sur l'hôte"
|
||||
tar -C "$GITHUB_WORKSPACE" --exclude=./.git -cf - . \
|
||||
| helper -i -v "${HOST_ROOT}/${PROJECT}:/dst" "$HELPER_IMAGE" tar -C /dst -xf -
|
||||
|
||||
echo "Réécriture de la configuration Compose"
|
||||
raw="${WORK_DIR}/rewrite.out"
|
||||
docker compose --ansi never \
|
||||
--project-name "$PROJECT" \
|
||||
--project-directory "$GITHUB_WORKSPACE" \
|
||||
-f "${GITHUB_WORKSPACE}/${file}" \
|
||||
config --format json \
|
||||
| helper -i \
|
||||
-e CI_PROJECT="$PROJECT" \
|
||||
-e CI_WORKSPACE="$GITHUB_WORKSPACE" \
|
||||
-e CI_HOST_DIR="${HOST_ROOT}/${PROJECT}" \
|
||||
-e CI_MEM_LIMIT="${CI_MEM_LIMIT:-}" \
|
||||
-e CI_CPU_LIMIT="${CI_CPU_LIMIT:-}" \
|
||||
-e CI_PIDS_LIMIT="${CI_PIDS_LIMIT:-}" \
|
||||
-v "${HOST_ROOT}/${PROJECT}:/ci" \
|
||||
"$HELPER_IMAGE" python3 -c "$(<"${SCRIPT_DIR}/compose-ci-rewrite.py")" \
|
||||
> "$raw"
|
||||
|
||||
sed -n '1p' "$raw" > "$COUNT_FILE"
|
||||
sed -n '2p' "$raw" > "$CI_FILE"
|
||||
sed -n '3,$p' "$raw" > "$MASK_FILE"
|
||||
rm -f "$raw"
|
||||
chmod 0600 "$CI_FILE" "$MASK_FILE"
|
||||
|
||||
load_state
|
||||
assert_isolated
|
||||
compose config --quiet
|
||||
echo "Configuration de CI validée (${SERVICE_COUNT} service(s))"
|
||||
}
|
||||
|
||||
cmd_up() {
|
||||
load_state
|
||||
if is_dry_run; then
|
||||
echo "CI_DRY_RUN=true : aucun conteneur démarré"
|
||||
return 0
|
||||
fi
|
||||
if (( SERVICE_COUNT == 0 )); then
|
||||
echo "Aucun service testable dans ce dépôt : rien à démarrer"
|
||||
return 0
|
||||
fi
|
||||
compose up --detach --build --quiet-pull
|
||||
}
|
||||
|
||||
evaluate_state() {
|
||||
local ids id name status code health verdict
|
||||
STATE_PENDING=0
|
||||
STATE_FAILED=0
|
||||
STATE_TABLE="$(printf '%-50s %-11s %-5s %-10s %s' CONTENEUR ETAT CODE SANTE VERDICT)"$'\n'
|
||||
|
||||
mapfile -t ids < <(compose ps --all --quiet | grep -v '^$' || true)
|
||||
if (( ${#ids[@]} == 0 )); then
|
||||
STATE_FAILED=1
|
||||
STATE_TABLE+="aucun conteneur créé"$'\n'
|
||||
return
|
||||
fi
|
||||
|
||||
for id in "${ids[@]}"; do
|
||||
IFS='|' read -r name status code health < <(
|
||||
docker inspect --format \
|
||||
'{{.Name}}|{{.State.Status}}|{{.State.ExitCode}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}-{{end}}' \
|
||||
"$id"
|
||||
)
|
||||
case "${status}:${health}" in
|
||||
running:- | running:healthy)
|
||||
verdict="OK" ;;
|
||||
running:starting | running:unhealthy | created:* | restarting:*)
|
||||
verdict="ATTENTE"; STATE_PENDING=1 ;;
|
||||
exited:*)
|
||||
if [[ "$code" == "0" ]]; then
|
||||
verdict="TERMINÉ"
|
||||
else
|
||||
verdict="ÉCHEC"; STATE_FAILED=1
|
||||
fi ;;
|
||||
*)
|
||||
verdict="ÉCHEC"; STATE_FAILED=1 ;;
|
||||
esac
|
||||
STATE_TABLE+="$(printf '%-50s %-11s %-5s %-10s %s' "${name#/}" "$status" "$code" "$health" "$verdict")"$'\n'
|
||||
done
|
||||
}
|
||||
|
||||
cmd_verify() {
|
||||
local timeout deadline stable=0
|
||||
load_state
|
||||
if is_dry_run || (( SERVICE_COUNT == 0 )); then
|
||||
echo "Rien à vérifier"
|
||||
return 0
|
||||
fi
|
||||
|
||||
timeout="${CI_WAIT_TIMEOUT:-300}"
|
||||
[[ "$timeout" =~ ^[0-9]+$ ]] || die "CI_WAIT_TIMEOUT invalide"
|
||||
deadline=$(( SECONDS + timeout ))
|
||||
|
||||
while :; do
|
||||
evaluate_state
|
||||
if (( STATE_FAILED )); then
|
||||
printf '%s' "$STATE_TABLE"
|
||||
die "au moins un conteneur est en échec"
|
||||
fi
|
||||
if (( ! STATE_PENDING )); then
|
||||
if (( stable )); then
|
||||
printf '%s' "$STATE_TABLE"
|
||||
echo "Tous les conteneurs sont démarrés et stables"
|
||||
return 0
|
||||
fi
|
||||
stable=1
|
||||
sleep "$STABILIZATION_SECONDS"
|
||||
continue
|
||||
fi
|
||||
stable=0
|
||||
if (( SECONDS >= deadline )); then
|
||||
printf '%s' "$STATE_TABLE"
|
||||
die "délai de ${timeout} s dépassé avant que tous les conteneurs soient prêts"
|
||||
fi
|
||||
sleep "$POLL_SECONDS"
|
||||
done
|
||||
}
|
||||
|
||||
cmd_logs() {
|
||||
local lines logs encoded secret
|
||||
if [[ ! -f "$CI_FILE" ]]; then
|
||||
echo "Pas de configuration de CI : aucun journal à afficher"
|
||||
return 0
|
||||
fi
|
||||
load_state
|
||||
lines="${CI_LOG_LINES:-50}"
|
||||
[[ "$lines" =~ ^[0-9]+$ ]] || die "CI_LOG_LINES invalide"
|
||||
|
||||
compose ps --all --format 'table {{.Service}}\t{{.State}}\t{{.Status}}' || true
|
||||
logs="$(compose logs --no-color --tail "$lines" 2>&1 || true)"
|
||||
|
||||
if [[ -s "$MASK_FILE" ]]; then
|
||||
while IFS= read -r encoded; do
|
||||
[[ -n "$encoded" ]] || continue
|
||||
secret="$(printf '%s' "$encoded" | base64 -d)"
|
||||
[[ -n "$secret" ]] && logs="${logs//"$secret"/***}"
|
||||
done < "$MASK_FILE"
|
||||
fi
|
||||
printf '%s\n' "$logs"
|
||||
}
|
||||
|
||||
remove_by_label() {
|
||||
local kind="$1" filter="label=com.docker.compose.project=${PROJECT}"
|
||||
case "$kind" in
|
||||
container) docker ps -aq --filter "$filter" | xargs -r docker rm -f >/dev/null ;;
|
||||
network) docker network ls -q --filter "$filter" | xargs -r docker network rm >/dev/null ;;
|
||||
volume) docker volume ls -q --filter "$filter" | xargs -r docker volume rm >/dev/null ;;
|
||||
esac
|
||||
}
|
||||
|
||||
cmd_cleanup() {
|
||||
local markers foreign status=0
|
||||
PROJECT="$(project_name)"
|
||||
|
||||
markers="$(docker ps -a \
|
||||
--filter "label=com.docker.compose.project=${PROJECT}" \
|
||||
--format "[{{.Label \"${CI_MARKER_LABEL}\"}}]")" \
|
||||
|| die "impossible de lister les conteneurs du projet ${PROJECT}"
|
||||
foreign="$(grep -cvxF '[true]' <<<"$markers" || true)"
|
||||
[[ -n "$markers" ]] || foreign=0
|
||||
if (( foreign > 0 )); then
|
||||
die "${foreign} conteneur(s) du projet ${PROJECT} sans marqueur de CI : nettoyage annulé"
|
||||
fi
|
||||
|
||||
if [[ -f "$CI_FILE" ]]; then
|
||||
compose down --volumes --remove-orphans --rmi local --timeout 30 || status=1
|
||||
fi
|
||||
remove_by_label container || status=1
|
||||
remove_by_label network || status=1
|
||||
remove_by_label volume || status=1
|
||||
|
||||
# shellcheck disable=SC2016 # $1 est développé par le sh du conteneur
|
||||
helper -v "${HOST_ROOT}:/ci-root" "$HELPER_IMAGE" \
|
||||
sh -c 'rm -rf -- "/ci-root/$1"' _ "$PROJECT" || status=1
|
||||
|
||||
if (( status )); then
|
||||
die "nettoyage incomplet du projet ${PROJECT}"
|
||||
fi
|
||||
echo "Projet ${PROJECT} nettoyé"
|
||||
}
|
||||
|
||||
main() {
|
||||
[[ -n "${GITHUB_WORKSPACE:-}" ]] || die "GITHUB_WORKSPACE absent"
|
||||
case "${1:-}" in
|
||||
prepare) cmd_prepare ;;
|
||||
up) cmd_up ;;
|
||||
verify) cmd_verify ;;
|
||||
logs) cmd_logs ;;
|
||||
cleanup) cmd_cleanup ;;
|
||||
*) die "usage : $0 prepare|up|verify|logs|cleanup" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in new issue
Block a user