new config
This commit is contained in:
@@ -0,0 +1,447 @@
|
||||
"""Réécrit une configuration Compose (sortie de `docker compose config --format json`)
|
||||
pour qu'elle puisse tourner en CI sur le même démon Docker que la production,
|
||||
sans jamais la toucher.
|
||||
|
||||
Exécuté dans un conteneur utilitaire lancé par compose-ci.sh :
|
||||
- stdin : configuration Compose résolue (JSON, contient des secrets : jamais affichée)
|
||||
- stdout : ligne 1 = nombre de services retenus
|
||||
ligne 2 = configuration réécrite (JSON sur une ligne)
|
||||
lignes suivantes = valeurs sensibles à masquer dans les journaux (base64)
|
||||
- stderr : rapport lisible (aucune valeur sensible)
|
||||
- /ci : copie du dépôt sur l'hôte (répertoire CI_HOST_DIR vu depuis ce conteneur)
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
import re
|
||||
import sys
|
||||
|
||||
LOCAL_DIR = "/ci"
|
||||
|
||||
PROJECT_RE = re.compile(r"^ci-[a-z0-9]([a-z0-9-]*[a-z0-9])?-[0-9]+-[0-9]+$")
|
||||
|
||||
# Limites appliquées aux services qui n'en déclarent pas (identiques pour tous les dépôts).
|
||||
MEM_LIMIT = "1g"
|
||||
CPU_LIMIT = 1.0
|
||||
PIDS_LIMIT = 2048
|
||||
|
||||
# Labels lus par des composants de production (routage, découverte, mises à jour).
|
||||
DROPPED_LABEL_PREFIXES = (
|
||||
"traefik.",
|
||||
"homepage.",
|
||||
"com.centurylinklabs.watchtower.",
|
||||
"sablier.",
|
||||
)
|
||||
# Cible du test HTTP, déduite des labels Traefik avant leur suppression.
|
||||
HTTP_TARGET_LABEL = "tips-of-mine.ci.http"
|
||||
HTTP_PORT_RE = re.compile(r"^traefik\.http\.services\.([^.]+)\.loadbalancer\.server\.port$")
|
||||
HTTP_SCHEME_RE = re.compile(r"^traefik\.http\.services\.([^.]+)\.loadbalancer\.server\.scheme$")
|
||||
CI_LABELS = {
|
||||
"traefik.enable": "false",
|
||||
"com.centurylinklabs.watchtower.enable": "false",
|
||||
"tips-of-mine.ci": "true",
|
||||
}
|
||||
|
||||
# Capacités non isolées par les namespaces : elles agissent sur l'hôte.
|
||||
DANGEROUS_CAPS = {
|
||||
"ALL", "SYS_ADMIN", "SYS_MODULE", "SYS_RAWIO", "SYS_TIME", "SYS_BOOT",
|
||||
"SYSLOG", "DAC_READ_SEARCH", "MAC_ADMIN", "MAC_OVERRIDE", "BPF", "PERFMON",
|
||||
}
|
||||
HOST_NAMESPACE_KEYS = ("pid", "ipc", "uts", "userns_mode", "cgroup")
|
||||
|
||||
# Répertoires hôtes qui contiennent (ou sont parents de) sockets de démons.
|
||||
SOCKET_PARENTS = {"/", "/run", "/var", "/var/run"}
|
||||
|
||||
SENSITIVE_NAME_RE = re.compile(
|
||||
r"PASS|PWD|SECRET|TOKEN|KEY|CREDENTIAL|PRIVATE|AUTH|DSN|URI|URL|SALT|COOKIE",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
MASK_MIN_LENGTH = 6
|
||||
OOM_SCORE_ADJ = 500
|
||||
|
||||
|
||||
def fail(message):
|
||||
print(f"ERREUR : {message}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def report(message):
|
||||
print(message, file=sys.stderr)
|
||||
|
||||
|
||||
def read_settings():
|
||||
project = os.environ.get("CI_PROJECT", "")
|
||||
workspace = os.environ.get("CI_WORKSPACE", "")
|
||||
host_dir = os.environ.get("CI_HOST_DIR", "")
|
||||
|
||||
if not PROJECT_RE.match(project):
|
||||
fail(f"nom de projet refusé : {project!r}")
|
||||
if not workspace.startswith("/"):
|
||||
fail("CI_WORKSPACE doit être un chemin absolu")
|
||||
if host_dir != f"/var/tmp/gitea-ci/{project}":
|
||||
fail(f"répertoire hôte refusé : {host_dir!r}")
|
||||
|
||||
return {
|
||||
"project": project,
|
||||
"workspace": posixpath.normpath(workspace),
|
||||
"host_dir": host_dir,
|
||||
"mem": MEM_LIMIT,
|
||||
"cpus": CPU_LIMIT,
|
||||
"pids": PIDS_LIMIT,
|
||||
}
|
||||
|
||||
|
||||
def is_under(path, root):
|
||||
path = posixpath.normpath(path)
|
||||
return path == root or path.startswith(root + "/")
|
||||
|
||||
|
||||
def relative_to_workspace(path, settings):
|
||||
return posixpath.relpath(posixpath.normpath(path), settings["workspace"])
|
||||
|
||||
|
||||
def host_path_problem(path, read_only):
|
||||
"""Raison d'exclusion d'un montage hôte situé hors du dépôt, ou None."""
|
||||
path = posixpath.normpath(path)
|
||||
if path.endswith(".sock") or "docker.sock" in path:
|
||||
return "socket de l'hôte"
|
||||
if path in SOCKET_PARENTS or path.startswith(("/run/", "/var/run/")):
|
||||
return "répertoire de sockets de l'hôte"
|
||||
if path == "/dev" or path.startswith("/dev/"):
|
||||
return "périphérique de l'hôte"
|
||||
if not read_only:
|
||||
return "montage de l'hôte en écriture"
|
||||
return None
|
||||
|
||||
|
||||
def copied_path_problem(path, settings):
|
||||
"""Refuse un chemin du dépôt qui sort de la copie via un lien symbolique."""
|
||||
rel = relative_to_workspace(path, settings)
|
||||
local = posixpath.normpath(posixpath.join(LOCAL_DIR, rel))
|
||||
if not is_under(os.path.realpath(local), LOCAL_DIR):
|
||||
return f"lien symbolique sortant du dépôt ({rel})"
|
||||
return None
|
||||
|
||||
|
||||
def file_resource_problem(resource, settings):
|
||||
"""Contrôle un secret ou une config de premier niveau déclaré par fichier."""
|
||||
source = (resource or {}).get("file")
|
||||
if not source:
|
||||
return None
|
||||
if is_under(source, settings["workspace"]):
|
||||
return copied_path_problem(source, settings)
|
||||
return host_path_problem(source, read_only=True)
|
||||
|
||||
|
||||
def service_problems(svc, settings, unsafe_secrets, unsafe_configs):
|
||||
reasons = []
|
||||
if svc.get("privileged"):
|
||||
reasons.append("privileged")
|
||||
for key in HOST_NAMESPACE_KEYS:
|
||||
if svc.get(key) == "host":
|
||||
reasons.append(f"{key}: host")
|
||||
network_mode = svc.get("network_mode") or ""
|
||||
if network_mode == "host" or network_mode.startswith("container:"):
|
||||
reasons.append(f"network_mode: {network_mode}")
|
||||
|
||||
reservations = ((svc.get("deploy") or {}).get("resources") or {}).get("reservations") or {}
|
||||
if svc.get("devices") or svc.get("device_cgroup_rules") or svc.get("gpus") or reservations.get("devices"):
|
||||
reasons.append("accès à des périphériques")
|
||||
|
||||
caps = {str(cap).upper().removeprefix("CAP_") for cap in svc.get("cap_add") or []}
|
||||
if caps & DANGEROUS_CAPS:
|
||||
reasons.append("cap_add " + ",".join(sorted(caps & DANGEROUS_CAPS)))
|
||||
|
||||
for entry in svc.get("volumes_from") or []:
|
||||
if str(entry).startswith("container:"):
|
||||
reasons.append("volumes_from vers un conteneur externe")
|
||||
|
||||
for volume in svc.get("volumes") or []:
|
||||
if volume.get("type") != "bind":
|
||||
continue
|
||||
source = volume.get("source") or ""
|
||||
if is_under(source, settings["workspace"]):
|
||||
problem = copied_path_problem(source, settings)
|
||||
else:
|
||||
problem = host_path_problem(source, bool(volume.get("read_only")))
|
||||
if problem:
|
||||
problem = f"{problem} ({source})"
|
||||
if problem:
|
||||
reasons.append(problem)
|
||||
|
||||
for ref in svc.get("secrets") or []:
|
||||
if ref.get("source") in unsafe_secrets:
|
||||
reasons.append(f"secret {ref.get('source')} non isolable")
|
||||
for ref in svc.get("configs") or []:
|
||||
if ref.get("source") in unsafe_configs:
|
||||
reasons.append(f"config {ref.get('source')} non isolable")
|
||||
return reasons
|
||||
|
||||
|
||||
def dependency_targets(svc):
|
||||
"""Services dont celui-ci ne peut pas se passer (réseau ou volumes partagés)."""
|
||||
targets = []
|
||||
network_mode = svc.get("network_mode") or ""
|
||||
if network_mode.startswith("service:"):
|
||||
targets.append(network_mode.removeprefix("service:"))
|
||||
for entry in svc.get("volumes_from") or []:
|
||||
entry = str(entry)
|
||||
if not entry.startswith("container:"):
|
||||
targets.append(entry.removeprefix("service:").split(":")[0])
|
||||
return targets
|
||||
|
||||
|
||||
def select_services(cfg, settings):
|
||||
services = cfg.get("services") or {}
|
||||
unsafe_secrets = {
|
||||
name for name, res in (cfg.get("secrets") or {}).items()
|
||||
if file_resource_problem(res, settings)
|
||||
}
|
||||
unsafe_configs = {
|
||||
name for name, res in (cfg.get("configs") or {}).items()
|
||||
if file_resource_problem(res, settings)
|
||||
}
|
||||
|
||||
excluded = {}
|
||||
for name, svc in services.items():
|
||||
reasons = service_problems(svc, settings, unsafe_secrets, unsafe_configs)
|
||||
if reasons:
|
||||
excluded[name] = reasons
|
||||
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for name, svc in services.items():
|
||||
if name in excluded:
|
||||
continue
|
||||
missing = sorted({t for t in dependency_targets(svc) if t in excluded})
|
||||
if missing:
|
||||
excluded[name] = [f"dépend du service exclu {', '.join(missing)}"]
|
||||
changed = True
|
||||
return excluded
|
||||
|
||||
|
||||
def collect_masks(services):
|
||||
masks = set()
|
||||
for svc in services.values():
|
||||
environment = svc.get("environment") or {}
|
||||
if isinstance(environment, list):
|
||||
environment = dict(item.split("=", 1) for item in environment if "=" in item)
|
||||
for key, value in environment.items():
|
||||
if value is None or not SENSITIVE_NAME_RE.search(key):
|
||||
continue
|
||||
value = str(value)
|
||||
# `compose config` échappe les $ en $$ ; le conteneur voit la forme simple.
|
||||
for variant in {value, value.replace("$$", "$")}:
|
||||
if len(variant) >= MASK_MIN_LENGTH:
|
||||
masks.add(variant)
|
||||
return masks
|
||||
|
||||
|
||||
def service_owner(svc):
|
||||
"""uid/gid numériques du `user:` du service, ou None s'ils ne sont pas numériques."""
|
||||
user = str(svc.get("user") or "")
|
||||
uid, _, gid = user.partition(":")
|
||||
if not uid.isdigit() or (gid and not gid.isdigit()):
|
||||
return None
|
||||
return int(uid), int(gid) if gid else int(uid)
|
||||
|
||||
|
||||
def chown_tree(local, owner):
|
||||
"""Attribue une copie du dépôt à l'utilisateur du service, sans suivre les liens."""
|
||||
uid, gid = owner
|
||||
os.lchown(local, uid, gid)
|
||||
if os.path.isdir(local) and not os.path.islink(local):
|
||||
for root, dirs, files in os.walk(local, followlinks=False):
|
||||
for entry in dirs + files:
|
||||
os.lchown(posixpath.join(root, entry), uid, gid)
|
||||
|
||||
|
||||
def relocate_source(source, settings, create_missing, owner=None):
|
||||
rel = relative_to_workspace(source, settings)
|
||||
local = posixpath.normpath(posixpath.join(LOCAL_DIR, rel))
|
||||
if create_missing and not os.path.lexists(local):
|
||||
os.makedirs(local, exist_ok=True)
|
||||
os.chmod(local, 0o777)
|
||||
if owner and rel != ".":
|
||||
chown_tree(local, owner)
|
||||
if rel == ".":
|
||||
return settings["host_dir"]
|
||||
return posixpath.join(settings["host_dir"], rel)
|
||||
|
||||
|
||||
def http_target(labels):
|
||||
"""Cibles HTTP d'un service exposé par Traefik : "http:8200,https:8443", "auto" ou None."""
|
||||
if not any(key.startswith("traefik.http.") for key in labels):
|
||||
return None
|
||||
if str(labels.get("traefik.enable", "")).strip().lower() == "false":
|
||||
return None
|
||||
ports, schemes = {}, {}
|
||||
for key, value in labels.items():
|
||||
match = HTTP_PORT_RE.match(key)
|
||||
if match:
|
||||
ports[match.group(1)] = str(value).strip()
|
||||
match = HTTP_SCHEME_RE.match(key)
|
||||
if match:
|
||||
schemes[match.group(1)] = str(value).strip().lower()
|
||||
targets = set()
|
||||
for name, port in ports.items():
|
||||
if port.isdigit():
|
||||
scheme = "https" if schemes.get(name) == "https" else "http"
|
||||
targets.add(f"{scheme}:{port}")
|
||||
return ",".join(sorted(targets)) if targets else "auto"
|
||||
|
||||
|
||||
def rewrite_labels(labels, project):
|
||||
if isinstance(labels, list):
|
||||
labels = dict(item.split("=", 1) if "=" in item else (item, "") for item in labels)
|
||||
labels = labels or {}
|
||||
target = http_target(labels)
|
||||
kept = {k: v for k, v in labels.items() if not k.startswith(DROPPED_LABEL_PREFIXES)}
|
||||
kept.update(CI_LABELS)
|
||||
kept["tips-of-mine.ci.project"] = project
|
||||
if target:
|
||||
kept[HTTP_TARGET_LABEL] = target
|
||||
return kept
|
||||
|
||||
|
||||
def apply_limits(svc, settings):
|
||||
deploy = svc.get("deploy") or {}
|
||||
deploy.pop("restart_policy", None)
|
||||
limits = (deploy.get("resources") or {}).get("limits")
|
||||
|
||||
if limits is not None:
|
||||
# Compose refuse deux valeurs distinctes (ex. cpus et deploy...limits.cpus) :
|
||||
# on complète le bloc déjà déclaré par le service.
|
||||
if not svc.get("mem_limit") and not limits.get("memory"):
|
||||
limits["memory"] = settings["mem"]
|
||||
if not svc.get("cpus") and not limits.get("cpus"):
|
||||
limits["cpus"] = str(settings["cpus"])
|
||||
if not svc.get("pids_limit") and not limits.get("pids"):
|
||||
limits["pids"] = settings["pids"]
|
||||
else:
|
||||
svc.setdefault("mem_limit", settings["mem"])
|
||||
svc.setdefault("cpus", settings["cpus"])
|
||||
svc.setdefault("pids_limit", settings["pids"])
|
||||
|
||||
# Pas de swap pour la CI : le swap de l'hôte sert à la production.
|
||||
memory = svc.get("mem_limit") or (limits or {}).get("memory")
|
||||
if memory and not svc.get("memswap_limit"):
|
||||
svc["memswap_limit"] = memory
|
||||
svc["oom_score_adj"] = max(int(svc.get("oom_score_adj") or 0), OOM_SCORE_ADJ)
|
||||
|
||||
if deploy:
|
||||
svc["deploy"] = deploy
|
||||
else:
|
||||
svc.pop("deploy", None)
|
||||
|
||||
|
||||
def rewrite_service(name, svc, excluded, settings):
|
||||
for key in ("container_name", "ports", "logging", "mac_address", "external_links"):
|
||||
svc.pop(key, None)
|
||||
if "build" in svc:
|
||||
# Image nommée <projet>-<service> : aucun tag de production n'est écrasé.
|
||||
svc.pop("image", None)
|
||||
elif svc.get("pull_policy") not in (None, "missing", "if_not_present", "never"):
|
||||
svc["pull_policy"] = "missing"
|
||||
svc["restart"] = "no"
|
||||
svc["labels"] = rewrite_labels(svc.get("labels"), settings["project"])
|
||||
apply_limits(svc, settings)
|
||||
|
||||
networks = svc.get("networks")
|
||||
if isinstance(networks, dict):
|
||||
for attachment in networks.values():
|
||||
if isinstance(attachment, dict):
|
||||
for key in ("ipv4_address", "ipv6_address", "link_local_ips", "mac_address"):
|
||||
attachment.pop(key, None)
|
||||
|
||||
depends_on = svc.get("depends_on")
|
||||
if isinstance(depends_on, dict):
|
||||
for target in [t for t in depends_on if t in excluded]:
|
||||
report(f" - {name} : dépendance vers {target} retirée (service exclu)")
|
||||
del depends_on[target]
|
||||
if not depends_on:
|
||||
svc.pop("depends_on")
|
||||
if svc.get("links"):
|
||||
svc["links"] = [l for l in svc["links"] if str(l).split(":")[0] not in excluded]
|
||||
if not svc["links"]:
|
||||
svc.pop("links")
|
||||
|
||||
# Les montages en écriture appartiennent à l'utilisateur du service, comme en
|
||||
# production : certaines images refusent de démarrer sinon (chown impossible).
|
||||
owner = service_owner(svc)
|
||||
relocated = 0
|
||||
for volume in svc.get("volumes") or []:
|
||||
if volume.get("type") == "bind" and is_under(volume.get("source") or "", settings["workspace"]):
|
||||
writable_owner = None if volume.get("read_only") else owner
|
||||
volume["source"] = relocate_source(
|
||||
volume["source"], settings, create_missing=True, owner=writable_owner
|
||||
)
|
||||
relocated += 1
|
||||
return relocated
|
||||
|
||||
|
||||
def rewrite_top_level(cfg, settings):
|
||||
project = settings["project"]
|
||||
for key, network in list((cfg.get("networks") or {}).items()):
|
||||
network = network or {}
|
||||
for field in ("external", "driver", "driver_opts", "ipam", "enable_ipv6"):
|
||||
network.pop(field, None)
|
||||
network["name"] = f"{project}_{key}"
|
||||
cfg["networks"][key] = network
|
||||
|
||||
for key, volume in list((cfg.get("volumes") or {}).items()):
|
||||
volume = volume or {}
|
||||
for field in ("external", "driver", "driver_opts"):
|
||||
volume.pop(field, None)
|
||||
volume["name"] = f"{project}_{key}"
|
||||
cfg["volumes"][key] = volume
|
||||
|
||||
for section in ("secrets", "configs"):
|
||||
for resource in (cfg.get(section) or {}).values():
|
||||
source = (resource or {}).get("file")
|
||||
if source and is_under(source, settings["workspace"]):
|
||||
resource["file"] = relocate_source(source, settings, create_missing=False)
|
||||
|
||||
cfg["name"] = project
|
||||
|
||||
|
||||
def main():
|
||||
settings = read_settings()
|
||||
try:
|
||||
cfg = json.load(sys.stdin)
|
||||
except json.JSONDecodeError:
|
||||
fail("configuration Compose illisible")
|
||||
|
||||
services = cfg.get("services") or {}
|
||||
if not services:
|
||||
fail("aucun service dans la configuration Compose")
|
||||
|
||||
masks = collect_masks(services)
|
||||
excluded = select_services(cfg, settings)
|
||||
for name in excluded:
|
||||
del services[name]
|
||||
|
||||
report(f"Projet de CI : {settings['project']}")
|
||||
report(f"Services testés : {', '.join(sorted(services)) or 'aucun'}")
|
||||
if excluded:
|
||||
report("Services exclus :")
|
||||
for name in sorted(excluded):
|
||||
report(f" - {name} : {'; '.join(excluded[name])}")
|
||||
|
||||
relocated = sum(rewrite_service(n, s, excluded, settings) for n, s in services.items())
|
||||
rewrite_top_level(cfg, settings)
|
||||
cfg["services"] = services
|
||||
report(f"Montages relocalisés : {relocated}")
|
||||
|
||||
out = sys.stdout
|
||||
out.write(f"{len(services)}\n")
|
||||
out.write(json.dumps(cfg, separators=(",", ":")) + "\n")
|
||||
for value in sorted(masks):
|
||||
out.write(base64.b64encode(value.encode()).decode() + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user