448 lines
16 KiB
Python
448 lines
16 KiB
Python
"""Réécrit une configuration Compose (sortie de `docker compose config --format json`)
|
|
pour qu'elle puisse tourner en CI sur le même démon Docker que la production,
|
|
sans jamais la toucher.
|
|
|
|
Exécuté dans un conteneur utilitaire lancé par compose-ci.sh :
|
|
- stdin : configuration Compose résolue (JSON, contient des secrets : jamais affichée)
|
|
- stdout : ligne 1 = nombre de services retenus
|
|
ligne 2 = configuration réécrite (JSON sur une ligne)
|
|
lignes suivantes = valeurs sensibles à masquer dans les journaux (base64)
|
|
- stderr : rapport lisible (aucune valeur sensible)
|
|
- /ci : copie du dépôt sur l'hôte (répertoire CI_HOST_DIR vu depuis ce conteneur)
|
|
"""
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
import posixpath
|
|
import re
|
|
import sys
|
|
|
|
LOCAL_DIR = "/ci"
|
|
|
|
PROJECT_RE = re.compile(r"^ci-[a-z0-9]([a-z0-9-]*[a-z0-9])?-[0-9]+-[0-9]+$")
|
|
|
|
# Limites appliquées aux services qui n'en déclarent pas (identiques pour tous les dépôts).
|
|
MEM_LIMIT = "1g"
|
|
CPU_LIMIT = 1.0
|
|
PIDS_LIMIT = 2048
|
|
|
|
# Labels lus par des composants de production (routage, découverte, mises à jour).
|
|
DROPPED_LABEL_PREFIXES = (
|
|
"traefik.",
|
|
"homepage.",
|
|
"com.centurylinklabs.watchtower.",
|
|
"sablier.",
|
|
)
|
|
# Cible du test HTTP, déduite des labels Traefik avant leur suppression.
|
|
HTTP_TARGET_LABEL = "tips-of-mine.ci.http"
|
|
HTTP_PORT_RE = re.compile(r"^traefik\.http\.services\.([^.]+)\.loadbalancer\.server\.port$")
|
|
HTTP_SCHEME_RE = re.compile(r"^traefik\.http\.services\.([^.]+)\.loadbalancer\.server\.scheme$")
|
|
CI_LABELS = {
|
|
"traefik.enable": "false",
|
|
"com.centurylinklabs.watchtower.enable": "false",
|
|
"tips-of-mine.ci": "true",
|
|
}
|
|
|
|
# Capacités non isolées par les namespaces : elles agissent sur l'hôte.
|
|
DANGEROUS_CAPS = {
|
|
"ALL", "SYS_ADMIN", "SYS_MODULE", "SYS_RAWIO", "SYS_TIME", "SYS_BOOT",
|
|
"SYSLOG", "DAC_READ_SEARCH", "MAC_ADMIN", "MAC_OVERRIDE", "BPF", "PERFMON",
|
|
}
|
|
HOST_NAMESPACE_KEYS = ("pid", "ipc", "uts", "userns_mode", "cgroup")
|
|
|
|
# Répertoires hôtes qui contiennent (ou sont parents de) sockets de démons.
|
|
SOCKET_PARENTS = {"/", "/run", "/var", "/var/run"}
|
|
|
|
SENSITIVE_NAME_RE = re.compile(
|
|
r"PASS|PWD|SECRET|TOKEN|KEY|CREDENTIAL|PRIVATE|AUTH|DSN|URI|URL|SALT|COOKIE",
|
|
re.IGNORECASE,
|
|
)
|
|
MASK_MIN_LENGTH = 6
|
|
OOM_SCORE_ADJ = 500
|
|
|
|
|
|
def fail(message):
|
|
print(f"ERREUR : {message}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
|
|
def report(message):
|
|
print(message, file=sys.stderr)
|
|
|
|
|
|
def read_settings():
|
|
project = os.environ.get("CI_PROJECT", "")
|
|
workspace = os.environ.get("CI_WORKSPACE", "")
|
|
host_dir = os.environ.get("CI_HOST_DIR", "")
|
|
|
|
if not PROJECT_RE.match(project):
|
|
fail(f"nom de projet refusé : {project!r}")
|
|
if not workspace.startswith("/"):
|
|
fail("CI_WORKSPACE doit être un chemin absolu")
|
|
if host_dir != f"/var/tmp/gitea-ci/{project}":
|
|
fail(f"répertoire hôte refusé : {host_dir!r}")
|
|
|
|
return {
|
|
"project": project,
|
|
"workspace": posixpath.normpath(workspace),
|
|
"host_dir": host_dir,
|
|
"mem": MEM_LIMIT,
|
|
"cpus": CPU_LIMIT,
|
|
"pids": PIDS_LIMIT,
|
|
}
|
|
|
|
|
|
def is_under(path, root):
|
|
path = posixpath.normpath(path)
|
|
return path == root or path.startswith(root + "/")
|
|
|
|
|
|
def relative_to_workspace(path, settings):
|
|
return posixpath.relpath(posixpath.normpath(path), settings["workspace"])
|
|
|
|
|
|
def host_path_problem(path, read_only):
|
|
"""Raison d'exclusion d'un montage hôte situé hors du dépôt, ou None."""
|
|
path = posixpath.normpath(path)
|
|
if path.endswith(".sock") or "docker.sock" in path:
|
|
return "socket de l'hôte"
|
|
if path in SOCKET_PARENTS or path.startswith(("/run/", "/var/run/")):
|
|
return "répertoire de sockets de l'hôte"
|
|
if path == "/dev" or path.startswith("/dev/"):
|
|
return "périphérique de l'hôte"
|
|
if not read_only:
|
|
return "montage de l'hôte en écriture"
|
|
return None
|
|
|
|
|
|
def copied_path_problem(path, settings):
|
|
"""Refuse un chemin du dépôt qui sort de la copie via un lien symbolique."""
|
|
rel = relative_to_workspace(path, settings)
|
|
local = posixpath.normpath(posixpath.join(LOCAL_DIR, rel))
|
|
if not is_under(os.path.realpath(local), LOCAL_DIR):
|
|
return f"lien symbolique sortant du dépôt ({rel})"
|
|
return None
|
|
|
|
|
|
def file_resource_problem(resource, settings):
|
|
"""Contrôle un secret ou une config de premier niveau déclaré par fichier."""
|
|
source = (resource or {}).get("file")
|
|
if not source:
|
|
return None
|
|
if is_under(source, settings["workspace"]):
|
|
return copied_path_problem(source, settings)
|
|
return host_path_problem(source, read_only=True)
|
|
|
|
|
|
def service_problems(svc, settings, unsafe_secrets, unsafe_configs):
|
|
reasons = []
|
|
if svc.get("privileged"):
|
|
reasons.append("privileged")
|
|
for key in HOST_NAMESPACE_KEYS:
|
|
if svc.get(key) == "host":
|
|
reasons.append(f"{key}: host")
|
|
network_mode = svc.get("network_mode") or ""
|
|
if network_mode == "host" or network_mode.startswith("container:"):
|
|
reasons.append(f"network_mode: {network_mode}")
|
|
|
|
reservations = ((svc.get("deploy") or {}).get("resources") or {}).get("reservations") or {}
|
|
if svc.get("devices") or svc.get("device_cgroup_rules") or svc.get("gpus") or reservations.get("devices"):
|
|
reasons.append("accès à des périphériques")
|
|
|
|
caps = {str(cap).upper().removeprefix("CAP_") for cap in svc.get("cap_add") or []}
|
|
if caps & DANGEROUS_CAPS:
|
|
reasons.append("cap_add " + ",".join(sorted(caps & DANGEROUS_CAPS)))
|
|
|
|
for entry in svc.get("volumes_from") or []:
|
|
if str(entry).startswith("container:"):
|
|
reasons.append("volumes_from vers un conteneur externe")
|
|
|
|
for volume in svc.get("volumes") or []:
|
|
if volume.get("type") != "bind":
|
|
continue
|
|
source = volume.get("source") or ""
|
|
if is_under(source, settings["workspace"]):
|
|
problem = copied_path_problem(source, settings)
|
|
else:
|
|
problem = host_path_problem(source, bool(volume.get("read_only")))
|
|
if problem:
|
|
problem = f"{problem} ({source})"
|
|
if problem:
|
|
reasons.append(problem)
|
|
|
|
for ref in svc.get("secrets") or []:
|
|
if ref.get("source") in unsafe_secrets:
|
|
reasons.append(f"secret {ref.get('source')} non isolable")
|
|
for ref in svc.get("configs") or []:
|
|
if ref.get("source") in unsafe_configs:
|
|
reasons.append(f"config {ref.get('source')} non isolable")
|
|
return reasons
|
|
|
|
|
|
def dependency_targets(svc):
|
|
"""Services dont celui-ci ne peut pas se passer (réseau ou volumes partagés)."""
|
|
targets = []
|
|
network_mode = svc.get("network_mode") or ""
|
|
if network_mode.startswith("service:"):
|
|
targets.append(network_mode.removeprefix("service:"))
|
|
for entry in svc.get("volumes_from") or []:
|
|
entry = str(entry)
|
|
if not entry.startswith("container:"):
|
|
targets.append(entry.removeprefix("service:").split(":")[0])
|
|
return targets
|
|
|
|
|
|
def select_services(cfg, settings):
|
|
services = cfg.get("services") or {}
|
|
unsafe_secrets = {
|
|
name for name, res in (cfg.get("secrets") or {}).items()
|
|
if file_resource_problem(res, settings)
|
|
}
|
|
unsafe_configs = {
|
|
name for name, res in (cfg.get("configs") or {}).items()
|
|
if file_resource_problem(res, settings)
|
|
}
|
|
|
|
excluded = {}
|
|
for name, svc in services.items():
|
|
reasons = service_problems(svc, settings, unsafe_secrets, unsafe_configs)
|
|
if reasons:
|
|
excluded[name] = reasons
|
|
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for name, svc in services.items():
|
|
if name in excluded:
|
|
continue
|
|
missing = sorted({t for t in dependency_targets(svc) if t in excluded})
|
|
if missing:
|
|
excluded[name] = [f"dépend du service exclu {', '.join(missing)}"]
|
|
changed = True
|
|
return excluded
|
|
|
|
|
|
def collect_masks(services):
|
|
masks = set()
|
|
for svc in services.values():
|
|
environment = svc.get("environment") or {}
|
|
if isinstance(environment, list):
|
|
environment = dict(item.split("=", 1) for item in environment if "=" in item)
|
|
for key, value in environment.items():
|
|
if value is None or not SENSITIVE_NAME_RE.search(key):
|
|
continue
|
|
value = str(value)
|
|
# `compose config` échappe les $ en $$ ; le conteneur voit la forme simple.
|
|
for variant in {value, value.replace("$$", "$")}:
|
|
if len(variant) >= MASK_MIN_LENGTH:
|
|
masks.add(variant)
|
|
return masks
|
|
|
|
|
|
def service_owner(svc):
|
|
"""uid/gid numériques du `user:` du service, ou None s'ils ne sont pas numériques."""
|
|
user = str(svc.get("user") or "")
|
|
uid, _, gid = user.partition(":")
|
|
if not uid.isdigit() or (gid and not gid.isdigit()):
|
|
return None
|
|
return int(uid), int(gid) if gid else int(uid)
|
|
|
|
|
|
def chown_tree(local, owner):
|
|
"""Attribue une copie du dépôt à l'utilisateur du service, sans suivre les liens."""
|
|
uid, gid = owner
|
|
os.lchown(local, uid, gid)
|
|
if os.path.isdir(local) and not os.path.islink(local):
|
|
for root, dirs, files in os.walk(local, followlinks=False):
|
|
for entry in dirs + files:
|
|
os.lchown(posixpath.join(root, entry), uid, gid)
|
|
|
|
|
|
def relocate_source(source, settings, create_missing, owner=None):
|
|
rel = relative_to_workspace(source, settings)
|
|
local = posixpath.normpath(posixpath.join(LOCAL_DIR, rel))
|
|
if create_missing and not os.path.lexists(local):
|
|
os.makedirs(local, exist_ok=True)
|
|
os.chmod(local, 0o777)
|
|
if owner and rel != ".":
|
|
chown_tree(local, owner)
|
|
if rel == ".":
|
|
return settings["host_dir"]
|
|
return posixpath.join(settings["host_dir"], rel)
|
|
|
|
|
|
def http_target(labels):
|
|
"""Cibles HTTP d'un service exposé par Traefik : "http:8200,https:8443", "auto" ou None."""
|
|
if not any(key.startswith("traefik.http.") for key in labels):
|
|
return None
|
|
if str(labels.get("traefik.enable", "")).strip().lower() == "false":
|
|
return None
|
|
ports, schemes = {}, {}
|
|
for key, value in labels.items():
|
|
match = HTTP_PORT_RE.match(key)
|
|
if match:
|
|
ports[match.group(1)] = str(value).strip()
|
|
match = HTTP_SCHEME_RE.match(key)
|
|
if match:
|
|
schemes[match.group(1)] = str(value).strip().lower()
|
|
targets = set()
|
|
for name, port in ports.items():
|
|
if port.isdigit():
|
|
scheme = "https" if schemes.get(name) == "https" else "http"
|
|
targets.add(f"{scheme}:{port}")
|
|
return ",".join(sorted(targets)) if targets else "auto"
|
|
|
|
|
|
def rewrite_labels(labels, project):
|
|
if isinstance(labels, list):
|
|
labels = dict(item.split("=", 1) if "=" in item else (item, "") for item in labels)
|
|
labels = labels or {}
|
|
target = http_target(labels)
|
|
kept = {k: v for k, v in labels.items() if not k.startswith(DROPPED_LABEL_PREFIXES)}
|
|
kept.update(CI_LABELS)
|
|
kept["tips-of-mine.ci.project"] = project
|
|
if target:
|
|
kept[HTTP_TARGET_LABEL] = target
|
|
return kept
|
|
|
|
|
|
def apply_limits(svc, settings):
|
|
deploy = svc.get("deploy") or {}
|
|
deploy.pop("restart_policy", None)
|
|
limits = (deploy.get("resources") or {}).get("limits")
|
|
|
|
if limits is not None:
|
|
# Compose refuse deux valeurs distinctes (ex. cpus et deploy...limits.cpus) :
|
|
# on complète le bloc déjà déclaré par le service.
|
|
if not svc.get("mem_limit") and not limits.get("memory"):
|
|
limits["memory"] = settings["mem"]
|
|
if not svc.get("cpus") and not limits.get("cpus"):
|
|
limits["cpus"] = str(settings["cpus"])
|
|
if not svc.get("pids_limit") and not limits.get("pids"):
|
|
limits["pids"] = settings["pids"]
|
|
else:
|
|
svc.setdefault("mem_limit", settings["mem"])
|
|
svc.setdefault("cpus", settings["cpus"])
|
|
svc.setdefault("pids_limit", settings["pids"])
|
|
|
|
# Pas de swap pour la CI : le swap de l'hôte sert à la production.
|
|
memory = svc.get("mem_limit") or (limits or {}).get("memory")
|
|
if memory and not svc.get("memswap_limit"):
|
|
svc["memswap_limit"] = memory
|
|
svc["oom_score_adj"] = max(int(svc.get("oom_score_adj") or 0), OOM_SCORE_ADJ)
|
|
|
|
if deploy:
|
|
svc["deploy"] = deploy
|
|
else:
|
|
svc.pop("deploy", None)
|
|
|
|
|
|
def rewrite_service(name, svc, excluded, settings):
|
|
for key in ("container_name", "ports", "logging", "mac_address", "external_links"):
|
|
svc.pop(key, None)
|
|
if "build" in svc:
|
|
# Image nommée <projet>-<service> : aucun tag de production n'est écrasé.
|
|
svc.pop("image", None)
|
|
elif svc.get("pull_policy") not in (None, "missing", "if_not_present", "never"):
|
|
svc["pull_policy"] = "missing"
|
|
svc["restart"] = "no"
|
|
svc["labels"] = rewrite_labels(svc.get("labels"), settings["project"])
|
|
apply_limits(svc, settings)
|
|
|
|
networks = svc.get("networks")
|
|
if isinstance(networks, dict):
|
|
for attachment in networks.values():
|
|
if isinstance(attachment, dict):
|
|
for key in ("ipv4_address", "ipv6_address", "link_local_ips", "mac_address"):
|
|
attachment.pop(key, None)
|
|
|
|
depends_on = svc.get("depends_on")
|
|
if isinstance(depends_on, dict):
|
|
for target in [t for t in depends_on if t in excluded]:
|
|
report(f" - {name} : dépendance vers {target} retirée (service exclu)")
|
|
del depends_on[target]
|
|
if not depends_on:
|
|
svc.pop("depends_on")
|
|
if svc.get("links"):
|
|
svc["links"] = [l for l in svc["links"] if str(l).split(":")[0] not in excluded]
|
|
if not svc["links"]:
|
|
svc.pop("links")
|
|
|
|
# Les montages en écriture appartiennent à l'utilisateur du service, comme en
|
|
# production : certaines images refusent de démarrer sinon (chown impossible).
|
|
owner = service_owner(svc)
|
|
relocated = 0
|
|
for volume in svc.get("volumes") or []:
|
|
if volume.get("type") == "bind" and is_under(volume.get("source") or "", settings["workspace"]):
|
|
writable_owner = None if volume.get("read_only") else owner
|
|
volume["source"] = relocate_source(
|
|
volume["source"], settings, create_missing=True, owner=writable_owner
|
|
)
|
|
relocated += 1
|
|
return relocated
|
|
|
|
|
|
def rewrite_top_level(cfg, settings):
|
|
project = settings["project"]
|
|
for key, network in list((cfg.get("networks") or {}).items()):
|
|
network = network or {}
|
|
for field in ("external", "driver", "driver_opts", "ipam", "enable_ipv6"):
|
|
network.pop(field, None)
|
|
network["name"] = f"{project}_{key}"
|
|
cfg["networks"][key] = network
|
|
|
|
for key, volume in list((cfg.get("volumes") or {}).items()):
|
|
volume = volume or {}
|
|
for field in ("external", "driver", "driver_opts"):
|
|
volume.pop(field, None)
|
|
volume["name"] = f"{project}_{key}"
|
|
cfg["volumes"][key] = volume
|
|
|
|
for section in ("secrets", "configs"):
|
|
for resource in (cfg.get(section) or {}).values():
|
|
source = (resource or {}).get("file")
|
|
if source and is_under(source, settings["workspace"]):
|
|
resource["file"] = relocate_source(source, settings, create_missing=False)
|
|
|
|
cfg["name"] = project
|
|
|
|
|
|
def main():
|
|
settings = read_settings()
|
|
try:
|
|
cfg = json.load(sys.stdin)
|
|
except json.JSONDecodeError:
|
|
fail("configuration Compose illisible")
|
|
|
|
services = cfg.get("services") or {}
|
|
if not services:
|
|
fail("aucun service dans la configuration Compose")
|
|
|
|
masks = collect_masks(services)
|
|
excluded = select_services(cfg, settings)
|
|
for name in excluded:
|
|
del services[name]
|
|
|
|
report(f"Projet de CI : {settings['project']}")
|
|
report(f"Services testés : {', '.join(sorted(services)) or 'aucun'}")
|
|
if excluded:
|
|
report("Services exclus :")
|
|
for name in sorted(excluded):
|
|
report(f" - {name} : {'; '.join(excluded[name])}")
|
|
|
|
relocated = sum(rewrite_service(n, s, excluded, settings) for n, s in services.items())
|
|
rewrite_top_level(cfg, settings)
|
|
cfg["services"] = services
|
|
report(f"Montages relocalisés : {relocated}")
|
|
|
|
out = sys.stdout
|
|
out.write(f"{len(services)}\n")
|
|
out.write(json.dumps(cfg, separators=(",", ":")) + "\n")
|
|
for value in sorted(masks):
|
|
out.write(base64.b64encode(value.encode()).decode() + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|