"""Réécrit une configuration Compose (sortie de `docker compose config --format json`) pour qu'elle puisse tourner en CI sur le même démon Docker que la production, sans jamais la toucher. Exécuté dans un conteneur utilitaire lancé par compose-ci.sh : - stdin : configuration Compose résolue (JSON, contient des secrets : jamais affichée) - stdout : ligne 1 = nombre de services retenus ligne 2 = configuration réécrite (JSON sur une ligne) lignes suivantes = valeurs sensibles à masquer dans les journaux (base64) - stderr : rapport lisible (aucune valeur sensible) - /ci : copie du dépôt sur l'hôte (répertoire CI_HOST_DIR vu depuis ce conteneur) """ import base64 import json import os import posixpath import re import sys LOCAL_DIR = "/ci" PROJECT_RE = re.compile(r"^ci-[a-z0-9]([a-z0-9-]*[a-z0-9])?-[0-9]+-[0-9]+$") # Limites appliquées aux services qui n'en déclarent pas (identiques pour tous les dépôts). MEM_LIMIT = "1g" CPU_LIMIT = 1.0 PIDS_LIMIT = 2048 # Labels lus par des composants de production (routage, découverte, mises à jour). DROPPED_LABEL_PREFIXES = ( "traefik.", "homepage.", "com.centurylinklabs.watchtower.", "sablier.", ) # Cible du test HTTP, déduite des labels Traefik avant leur suppression. HTTP_TARGET_LABEL = "tips-of-mine.ci.http" HTTP_PORT_RE = re.compile(r"^traefik\.http\.services\.([^.]+)\.loadbalancer\.server\.port$") HTTP_SCHEME_RE = re.compile(r"^traefik\.http\.services\.([^.]+)\.loadbalancer\.server\.scheme$") CI_LABELS = { "traefik.enable": "false", "com.centurylinklabs.watchtower.enable": "false", "tips-of-mine.ci": "true", } # Capacités non isolées par les namespaces : elles agissent sur l'hôte. DANGEROUS_CAPS = { "ALL", "SYS_ADMIN", "SYS_MODULE", "SYS_RAWIO", "SYS_TIME", "SYS_BOOT", "SYSLOG", "DAC_READ_SEARCH", "MAC_ADMIN", "MAC_OVERRIDE", "BPF", "PERFMON", } HOST_NAMESPACE_KEYS = ("pid", "ipc", "uts", "userns_mode", "cgroup") # Répertoires hôtes qui contiennent (ou sont parents de) sockets de démons. SOCKET_PARENTS = {"/", "/run", "/var", "/var/run"} SENSITIVE_NAME_RE = re.compile( r"PASS|PWD|SECRET|TOKEN|KEY|CREDENTIAL|PRIVATE|AUTH|DSN|URI|URL|SALT|COOKIE", re.IGNORECASE, ) MASK_MIN_LENGTH = 6 OOM_SCORE_ADJ = 500 def fail(message): print(f"ERREUR : {message}", file=sys.stderr) sys.exit(1) def report(message): print(message, file=sys.stderr) def read_settings(): project = os.environ.get("CI_PROJECT", "") workspace = os.environ.get("CI_WORKSPACE", "") host_dir = os.environ.get("CI_HOST_DIR", "") if not PROJECT_RE.match(project): fail(f"nom de projet refusé : {project!r}") if not workspace.startswith("/"): fail("CI_WORKSPACE doit être un chemin absolu") if host_dir != f"/var/tmp/gitea-ci/{project}": fail(f"répertoire hôte refusé : {host_dir!r}") return { "project": project, "workspace": posixpath.normpath(workspace), "host_dir": host_dir, "mem": MEM_LIMIT, "cpus": CPU_LIMIT, "pids": PIDS_LIMIT, } def is_under(path, root): path = posixpath.normpath(path) return path == root or path.startswith(root + "/") def relative_to_workspace(path, settings): return posixpath.relpath(posixpath.normpath(path), settings["workspace"]) def host_path_problem(path, read_only): """Raison d'exclusion d'un montage hôte situé hors du dépôt, ou None.""" path = posixpath.normpath(path) if path.endswith(".sock") or "docker.sock" in path: return "socket de l'hôte" if path in SOCKET_PARENTS or path.startswith(("/run/", "/var/run/")): return "répertoire de sockets de l'hôte" if path == "/dev" or path.startswith("/dev/"): return "périphérique de l'hôte" if not read_only: return "montage de l'hôte en écriture" return None def copied_path_problem(path, settings): """Refuse un chemin du dépôt qui sort de la copie via un lien symbolique.""" rel = relative_to_workspace(path, settings) local = posixpath.normpath(posixpath.join(LOCAL_DIR, rel)) if not is_under(os.path.realpath(local), LOCAL_DIR): return f"lien symbolique sortant du dépôt ({rel})" return None def file_resource_problem(resource, settings): """Contrôle un secret ou une config de premier niveau déclaré par fichier.""" source = (resource or {}).get("file") if not source: return None if is_under(source, settings["workspace"]): return copied_path_problem(source, settings) return host_path_problem(source, read_only=True) def service_problems(svc, settings, unsafe_secrets, unsafe_configs): reasons = [] if svc.get("privileged"): reasons.append("privileged") for key in HOST_NAMESPACE_KEYS: if svc.get(key) == "host": reasons.append(f"{key}: host") network_mode = svc.get("network_mode") or "" if network_mode == "host" or network_mode.startswith("container:"): reasons.append(f"network_mode: {network_mode}") reservations = ((svc.get("deploy") or {}).get("resources") or {}).get("reservations") or {} if svc.get("devices") or svc.get("device_cgroup_rules") or svc.get("gpus") or reservations.get("devices"): reasons.append("accès à des périphériques") caps = {str(cap).upper().removeprefix("CAP_") for cap in svc.get("cap_add") or []} if caps & DANGEROUS_CAPS: reasons.append("cap_add " + ",".join(sorted(caps & DANGEROUS_CAPS))) for entry in svc.get("volumes_from") or []: if str(entry).startswith("container:"): reasons.append("volumes_from vers un conteneur externe") for volume in svc.get("volumes") or []: if volume.get("type") != "bind": continue source = volume.get("source") or "" if is_under(source, settings["workspace"]): problem = copied_path_problem(source, settings) else: problem = host_path_problem(source, bool(volume.get("read_only"))) if problem: problem = f"{problem} ({source})" if problem: reasons.append(problem) for ref in svc.get("secrets") or []: if ref.get("source") in unsafe_secrets: reasons.append(f"secret {ref.get('source')} non isolable") for ref in svc.get("configs") or []: if ref.get("source") in unsafe_configs: reasons.append(f"config {ref.get('source')} non isolable") return reasons def dependency_targets(svc): """Services dont celui-ci ne peut pas se passer (réseau ou volumes partagés).""" targets = [] network_mode = svc.get("network_mode") or "" if network_mode.startswith("service:"): targets.append(network_mode.removeprefix("service:")) for entry in svc.get("volumes_from") or []: entry = str(entry) if not entry.startswith("container:"): targets.append(entry.removeprefix("service:").split(":")[0]) return targets def select_services(cfg, settings): services = cfg.get("services") or {} unsafe_secrets = { name for name, res in (cfg.get("secrets") or {}).items() if file_resource_problem(res, settings) } unsafe_configs = { name for name, res in (cfg.get("configs") or {}).items() if file_resource_problem(res, settings) } excluded = {} for name, svc in services.items(): reasons = service_problems(svc, settings, unsafe_secrets, unsafe_configs) if reasons: excluded[name] = reasons changed = True while changed: changed = False for name, svc in services.items(): if name in excluded: continue missing = sorted({t for t in dependency_targets(svc) if t in excluded}) if missing: excluded[name] = [f"dépend du service exclu {', '.join(missing)}"] changed = True return excluded def collect_masks(services): masks = set() for svc in services.values(): environment = svc.get("environment") or {} if isinstance(environment, list): environment = dict(item.split("=", 1) for item in environment if "=" in item) for key, value in environment.items(): if value is None or not SENSITIVE_NAME_RE.search(key): continue value = str(value) # `compose config` échappe les $ en $$ ; le conteneur voit la forme simple. for variant in {value, value.replace("$$", "$")}: if len(variant) >= MASK_MIN_LENGTH: masks.add(variant) return masks def service_owner(svc): """uid/gid numériques du `user:` du service, ou None s'ils ne sont pas numériques.""" user = str(svc.get("user") or "") uid, _, gid = user.partition(":") if not uid.isdigit() or (gid and not gid.isdigit()): return None return int(uid), int(gid) if gid else int(uid) def chown_tree(local, owner): """Attribue une copie du dépôt à l'utilisateur du service, sans suivre les liens.""" uid, gid = owner os.lchown(local, uid, gid) if os.path.isdir(local) and not os.path.islink(local): for root, dirs, files in os.walk(local, followlinks=False): for entry in dirs + files: os.lchown(posixpath.join(root, entry), uid, gid) def relocate_source(source, settings, create_missing, owner=None): rel = relative_to_workspace(source, settings) local = posixpath.normpath(posixpath.join(LOCAL_DIR, rel)) if create_missing and not os.path.lexists(local): os.makedirs(local, exist_ok=True) os.chmod(local, 0o777) if owner and rel != ".": chown_tree(local, owner) if rel == ".": return settings["host_dir"] return posixpath.join(settings["host_dir"], rel) def http_target(labels): """Cibles HTTP d'un service exposé par Traefik : "http:8200,https:8443", "auto" ou None.""" if not any(key.startswith("traefik.http.") for key in labels): return None if str(labels.get("traefik.enable", "")).strip().lower() == "false": return None ports, schemes = {}, {} for key, value in labels.items(): match = HTTP_PORT_RE.match(key) if match: ports[match.group(1)] = str(value).strip() match = HTTP_SCHEME_RE.match(key) if match: schemes[match.group(1)] = str(value).strip().lower() targets = set() for name, port in ports.items(): if port.isdigit(): scheme = "https" if schemes.get(name) == "https" else "http" targets.add(f"{scheme}:{port}") return ",".join(sorted(targets)) if targets else "auto" def rewrite_labels(labels, project): if isinstance(labels, list): labels = dict(item.split("=", 1) if "=" in item else (item, "") for item in labels) labels = labels or {} target = http_target(labels) kept = {k: v for k, v in labels.items() if not k.startswith(DROPPED_LABEL_PREFIXES)} kept.update(CI_LABELS) kept["tips-of-mine.ci.project"] = project if target: kept[HTTP_TARGET_LABEL] = target return kept def apply_limits(svc, settings): deploy = svc.get("deploy") or {} deploy.pop("restart_policy", None) limits = (deploy.get("resources") or {}).get("limits") if limits is not None: # Compose refuse deux valeurs distinctes (ex. cpus et deploy...limits.cpus) : # on complète le bloc déjà déclaré par le service. if not svc.get("mem_limit") and not limits.get("memory"): limits["memory"] = settings["mem"] if not svc.get("cpus") and not limits.get("cpus"): limits["cpus"] = str(settings["cpus"]) if not svc.get("pids_limit") and not limits.get("pids"): limits["pids"] = settings["pids"] else: svc.setdefault("mem_limit", settings["mem"]) svc.setdefault("cpus", settings["cpus"]) svc.setdefault("pids_limit", settings["pids"]) # Pas de swap pour la CI : le swap de l'hôte sert à la production. memory = svc.get("mem_limit") or (limits or {}).get("memory") if memory and not svc.get("memswap_limit"): svc["memswap_limit"] = memory svc["oom_score_adj"] = max(int(svc.get("oom_score_adj") or 0), OOM_SCORE_ADJ) if deploy: svc["deploy"] = deploy else: svc.pop("deploy", None) def rewrite_service(name, svc, excluded, settings): for key in ("container_name", "ports", "logging", "mac_address", "external_links"): svc.pop(key, None) if "build" in svc: # Image nommée - : aucun tag de production n'est écrasé. svc.pop("image", None) elif svc.get("pull_policy") not in (None, "missing", "if_not_present", "never"): svc["pull_policy"] = "missing" svc["restart"] = "no" svc["labels"] = rewrite_labels(svc.get("labels"), settings["project"]) apply_limits(svc, settings) networks = svc.get("networks") if isinstance(networks, dict): for attachment in networks.values(): if isinstance(attachment, dict): for key in ("ipv4_address", "ipv6_address", "link_local_ips", "mac_address"): attachment.pop(key, None) depends_on = svc.get("depends_on") if isinstance(depends_on, dict): for target in [t for t in depends_on if t in excluded]: report(f" - {name} : dépendance vers {target} retirée (service exclu)") del depends_on[target] if not depends_on: svc.pop("depends_on") if svc.get("links"): svc["links"] = [l for l in svc["links"] if str(l).split(":")[0] not in excluded] if not svc["links"]: svc.pop("links") # Les montages en écriture appartiennent à l'utilisateur du service, comme en # production : certaines images refusent de démarrer sinon (chown impossible). owner = service_owner(svc) relocated = 0 for volume in svc.get("volumes") or []: if volume.get("type") == "bind" and is_under(volume.get("source") or "", settings["workspace"]): writable_owner = None if volume.get("read_only") else owner volume["source"] = relocate_source( volume["source"], settings, create_missing=True, owner=writable_owner ) relocated += 1 return relocated def rewrite_top_level(cfg, settings): project = settings["project"] for key, network in list((cfg.get("networks") or {}).items()): network = network or {} for field in ("external", "driver", "driver_opts", "ipam", "enable_ipv6"): network.pop(field, None) network["name"] = f"{project}_{key}" cfg["networks"][key] = network for key, volume in list((cfg.get("volumes") or {}).items()): volume = volume or {} for field in ("external", "driver", "driver_opts"): volume.pop(field, None) volume["name"] = f"{project}_{key}" cfg["volumes"][key] = volume for section in ("secrets", "configs"): for resource in (cfg.get(section) or {}).values(): source = (resource or {}).get("file") if source and is_under(source, settings["workspace"]): resource["file"] = relocate_source(source, settings, create_missing=False) cfg["name"] = project def main(): settings = read_settings() try: cfg = json.load(sys.stdin) except json.JSONDecodeError: fail("configuration Compose illisible") services = cfg.get("services") or {} if not services: fail("aucun service dans la configuration Compose") masks = collect_masks(services) excluded = select_services(cfg, settings) for name in excluded: del services[name] report(f"Projet de CI : {settings['project']}") report(f"Services testés : {', '.join(sorted(services)) or 'aucun'}") if excluded: report("Services exclus :") for name in sorted(excluded): report(f" - {name} : {'; '.join(excluded[name])}") relocated = sum(rewrite_service(n, s, excluded, settings) for n, s in services.items()) rewrite_top_level(cfg, settings) cfg["services"] = services report(f"Montages relocalisés : {relocated}") out = sys.stdout out.write(f"{len(services)}\n") out.write(json.dumps(cfg, separators=(",", ":")) + "\n") for value in sorted(masks): out.write(base64.b64encode(value.encode()).decode() + "\n") if __name__ == "__main__": main()