Coverage for src/footman/_globals.py: 90%
617 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-09-06 02:59 +0000
1"""Process-globals routers — the parallel regime made real for env (and kin).
3In the parallel regime nobody mutates process globals: cwd and env are
4context data (`ctx.cwd` / `ctx.env`), applied per child at spawn. The
5routers here make that regime hold for code that has never heard of it,
6the same way the output router does for `print()`:
8- **os.environ is virtualised for the run.** Reads see the run-start
9 snapshot plus the current task's overlay — exactly what the subprocess
10 branch of the same call would inject as `env=`, closing the
11 in-process/subprocess parity hole. Writes from a task body scope to the
12 task's own overlay (`ctx.env`): visible to its own reads and every child
13 it spawns, invisible to siblings — with a task-attributed teach-once
14 note naming the deliberate spelling. Deletion has no additive spelling,
15 so it is a taught error.
17The class methods of `os._Environ` are wrapped (never the object
18replaced), so every alias — `from os import environ`, `os.getenv` — is
19covered, and `os.environb` (a different instance) passes through
20untouched. Installed at the run boundary (`run_plan`), refcounted for
21nested runs, delegating to the originals outside a run — a bare
22`import footman` never pays for any of this.
24Originals are also re-exported (`real_getcwd`, …) so footman's own
25internals keep touching the real process state: the routers exist for
26*task* code, not for the framework.
27"""
29from __future__ import annotations
31import os
32import sys
33import threading
34import time
35from collections.abc import Callable
36from contextvars import ContextVar
37from typing import Any
39# Originals, captured at import time. Internals use these; the guards and
40# routers exist for task code, and footman warning about itself would be
41# noise with no lesson in it.
42real_getcwd = os.getcwd
43real_chdir = os.chdir
45_lock = threading.Lock()
46_installs = 0 # refcount: nested runs share one install
47_snapshot: dict[str, str] = {} # the run-start environment, pinned
48_environ_saved: dict[str, Any] = {} # the wrapped class's original methods
51def active() -> bool:
52 """Whether the routers are installed (a run is in flight)."""
53 return _installs > 0
56# Never inherited: the caller's interpreter is never a spawned tool's
57# interpreter. On Windows `uv run` exports PYTHONHOME pointing at the
58# environment it launched — footman's own — and every child inherits it, so a
59# console script belonging to any *other* Python loads that stdlib instead of
60# its own and dies before it can say anything ("Could not import
61# runpy._run_module_as_main"). Measured: 107 tools read as holes in one walk,
62# every one whose launcher was a console script rather than a native binary.
63# Confirmed on Windows under uv 0.11.1 and 0.11.31; on macOS and Linux the
64# variable is simply absent, so dropping it costs those platforms nothing.
65#
66# Dropped from the *inherited* copy, not at spawn: a task that deliberately
67# sets PYTHONHOME — or hands one to `run(env=…)` — still gets exactly what it
68# asked for. Inheritance is the thing nobody chose.
69#
70# PYTHONPATH is deliberately NOT here. People export it on purpose
71# (`PYTHONPATH=src`) meaning it for what their tasks run, it is harmless when
72# merely present (a directory that isn't on it changes nothing), and footman
73# cannot tell a deliberate one from an inherited one. The tool walk strips it
74# separately, where reading other-era venvs makes it genuinely dangerous.
75_NEVER_INHERITED = frozenset({"PYTHONHOME", "PYTHONEXECUTABLE"})
78def _inheritable(env: dict[str, str]) -> dict[str, str]:
79 """*env* minus the variables that describe our own interpreter."""
80 return {k: v for k, v in env.items() if _norm(k) not in _NEVER_INHERITED}
83def base_env() -> dict[str, str]:
84 """The environment a fresh task starts from — a real dict, never a diff.
86 Inside a run that is the pinned run-start snapshot, so every task begins
87 from the same base whatever a sibling has done to its own. Outside one it
88 is the live process environment, read through the *saved* method so an
89 installed router cannot answer about some other task's view.
91 Either way it arrives without `_NEVER_INHERITED`.
92 """
93 if _installs:
94 return _inheritable(_snapshot)
95 if "copy" in _environ_saved: # router installed but not for this read 95 ↛ 96line 95 didn't jump to line 96 because the condition on line 95 was never true
96 return _inheritable(_environ_saved["copy"](os.environ))
97 return _inheritable(dict(os.environ))
100def _norm(key: str) -> str:
101 """Windows environment lookups are case-insensitive; mirror that."""
102 return key.upper() if os.name == "nt" else key
105def _merged() -> dict[str, str]:
106 """The virtual environment: simply the task's own, which is a whole one.
108 It used to be `snapshot + overlay`, merged here and spelled two other ways
109 elsewhere. A task now owns its environment outright, so there is one value
110 and nothing to reconcile — which is what makes deletion ordinary.
111 """
112 from footman.context import current
114 env = current().env
115 if os.name == "nt":
116 return {k.upper(): v for k, v in env.items()}
117 return dict(env)
120def _note(kind: str, text: str) -> None:
121 """Fire a task-attributed note — the door every interception uses.
123 Levels, dedup, the site, the record and the policy all live in
124 `_notes.emit`; this stays as the one spelling the routers import."""
125 from footman import _notes
127 _notes.emit(kind, text)
130def _in_task() -> bool:
131 from footman.context import current
133 return current().in_task
136def _install_environ() -> None:
137 env_cls = type(os.environ)
138 names = (
139 "__getitem__",
140 "__setitem__",
141 "__delitem__",
142 "__iter__",
143 "__len__",
144 "__contains__",
145 "copy",
146 "setdefault",
147 )
148 for name in names:
149 _environ_saved[name] = getattr(env_cls, name)
150 # The originals, typed as the unbound methods they are — the wrappers
151 # below both call them and must honour their signatures.
152 orig_get: Callable[[Any, str], str] = _environ_saved["__getitem__"]
153 orig_set: Callable[[Any, str, str], None] = _environ_saved["__setitem__"]
154 orig_del: Callable[[Any, str], None] = _environ_saved["__delitem__"]
155 orig_iter: Callable[[Any], Any] = _environ_saved["__iter__"]
156 orig_len: Callable[[Any], int] = _environ_saved["__len__"]
157 orig_contains: Callable[[Any, object], bool] = _environ_saved["__contains__"]
158 orig_copy: Callable[[Any], Any] = _environ_saved["copy"]
159 orig_setdefault: Callable[[Any, str, str], str] = _environ_saved["setdefault"]
161 def _virtual(self: Any) -> bool:
162 if self is not os.environ or not _installs: 162 ↛ 163line 162 didn't jump to line 163 because the condition on line 162 was never true
163 return False
164 from footman.context import current
166 # A serial/exclusive task owns the real globals: pass through.
167 return not current().serial_active
169 def __getitem__(self: Any, key: str) -> str:
170 if not _virtual(self):
171 return orig_get(self, key)
172 return _merged()[_norm(key)]
174 def __setitem__(self: Any, key: str, value: str) -> None:
175 if not _virtual(self) or not _in_task():
176 return orig_set(self, key, value)
177 if not isinstance(key, str) or not isinstance(value, str): 177 ↛ 178line 177 didn't jump to line 178 because the condition on line 177 was never true
178 raise TypeError("str expected for environment keys and values")
179 from footman.context import current
181 ctx = current()
182 ctx.env[key] = value
183 # The variable is the instance: one note per variable, so every
184 # issue surfaces in one run and a known-harmless write can be
185 # classified by name without hiding the next variable.
186 _note(
187 f"environ-write:{_norm(key)}",
188 f"task {ctx.task or key} sets {key} via os.environ — footman "
189 f"scoped it to this task (children see it, siblings don't). "
190 f"Say it on purpose with env= / ctx.env.",
191 )
193 def __delitem__(self: Any, key: str) -> None:
194 if not _virtual(self) or not _in_task():
195 return orig_del(self, key)
196 from footman.context import current
198 # Ordinary, because a task owns its environment outright: the key goes
199 # from this task's copy and from every child it spawns after, while a
200 # sibling's copy is untouched. This used to be a taught error for any
201 # key the task had not set itself — the overlay had no way to say
202 # "absent" — which left presence-semantics variables (NO_COLOR, CI)
203 # impossible to hide from an in-process tool.
204 ctx = current()
205 hits = [k for k in ctx.env if _norm(k) == _norm(key)]
206 if not hits:
207 raise KeyError(key) # absent: Python's own answer
208 for k in hits:
209 del ctx.env[k]
211 def __iter__(self: Any) -> Any:
212 if not _virtual(self):
213 return orig_iter(self)
214 return iter(_merged())
216 def __len__(self: Any) -> int:
217 if not _virtual(self):
218 return orig_len(self)
219 return len(_merged())
221 def __contains__(self: Any, key: object) -> bool:
222 if not _virtual(self):
223 return orig_contains(self, key)
224 return isinstance(key, str) and _norm(key) in _merged()
226 def copy(self: Any) -> Any:
227 if not _virtual(self): 227 ↛ 228line 227 didn't jump to line 228 because the condition on line 227 was never true
228 return orig_copy(self)
229 return _merged()
231 def setdefault(self: Any, key: str, value: str) -> str:
232 if not _virtual(self): 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true
233 return orig_setdefault(self, key, value)
234 try:
235 found: str = self[key]
236 except KeyError:
237 self[key] = value
238 return value
239 return found
241 # Deliberate runtime patching — the router IS a monkeypatch, and no
242 # checker has a way to bless a static spelling of one (ty refuses even
243 # signature-identical patches, and which os.* names exist depends on
244 # the platform being checked). setattr says what is happening — a
245 # dynamic write — and needs no suppression on any checker or platform;
246 # ruff's B010 is excused for this file in pyproject.
247 setattr(env_cls, "__getitem__", __getitem__)
248 setattr(env_cls, "__setitem__", __setitem__)
249 setattr(env_cls, "__delitem__", __delitem__)
250 setattr(env_cls, "__iter__", __iter__)
251 setattr(env_cls, "__len__", __len__)
252 setattr(env_cls, "__contains__", __contains__)
253 setattr(env_cls, "copy", copy)
254 setattr(env_cls, "setdefault", setdefault)
257def _restore_environ() -> None:
258 env_cls = type(os.environ)
259 for name, orig in _environ_saved.items():
260 setattr(env_cls, name, orig)
261 _environ_saved.clear()
264# --- the Popen injection ------------------------------------------------------
266_popen_saved: Any = None
269def _managed_task() -> tuple[Any, bool]:
270 """(ctx, guarded) — guarded only inside a managed *parallel* task body.
272 `unmanaged` is the one off-switch (that token *means* footman stays
273 out), and a serial/exclusive task owns the real globals legitimately.
274 Work footman does on the body's behalf (`internal()`) is never guarded
275 either: it has already worked out whatever a guard would fill in, and a
276 note about it would name a line the author never wrote."""
277 from footman.context import current
279 ctx = current()
280 guarded = (
281 bool(_installs)
282 and not _is_internal()
283 and ctx.in_task
284 and not ctx.cwd_unmanaged
285 and not ctx.serial_active
286 )
287 return ctx, guarded
290def _install_popen() -> None:
291 import subprocess
293 global _popen_saved
294 _popen_saved = subprocess.Popen.__init__
295 orig = _popen_saved
297 def __init__(self: Any, args: Any, *pa: Any, **kw: Any) -> None:
298 ctx, guarded = _managed_task()
299 # Inject only for a managed parallel task, only into kwargs left at
300 # their defaults, and never when positionals reach as far as cwd
301 # (the 10-positional spelling is antique; leave it alone entirely).
302 if guarded and len(pa) < 9:
303 filled = []
304 if kw.get("cwd") is None and ctx.cwd is not None:
305 kw["cwd"] = ctx.cwd
306 filled.append("cwd")
307 if kw.get("env") is None:
308 kw["env"] = dict(ctx.env) # the task's own, handed over whole
309 filled.append("env")
310 if filled:
311 from footman import _notes
313 _note(
314 f"popen-inject:{_notes.program_name(args)}",
315 f"task {ctx.task or '?'} spawns via raw subprocess — "
316 f"footman filled in {' and '.join(filled)} from the task "
317 f"context. Prefer run() for capture and reporting, or "
318 f"pass cwd=/env= to make it deliberate.",
319 )
320 orig(self, args, *pa, **kw)
322 setattr(subprocess.Popen, "__init__", __init__)
325def _restore_popen() -> None:
326 global _popen_saved
327 if _popen_saved is not None: 327 ↛ exitline 327 didn't return from function '_restore_popen' because the condition on line 327 was always true
328 import subprocess
330 setattr(subprocess.Popen, "__init__", _popen_saved)
331 _popen_saved = None
334# --- the os guards ------------------------------------------------------------
336_guard_saved: dict[str, Any] = {}
339_tempdir_warmed = False
340# Set while footman itself is the one asking a guarded question. Thread-local
341# on purpose: a sibling task reading the cwd for real, in its own thread, still
342# earns its note while this one is resolving.
343_internal = threading.local()
346def _is_internal() -> bool:
347 return bool(getattr(_internal, "on", False))
350def internal() -> Any:
351 """Mark this thread as footman working on the body's behalf, so the
352 guards and the Popen injector stay out of the way. Saves and restores
353 rather than clearing, so a nested region leaves the outer one standing."""
354 import contextlib
356 @contextlib.contextmanager
357 def _internal_cm() -> Any:
358 was = _is_internal()
359 _internal.on = True
360 try:
361 yield
362 finally:
363 _internal.on = was
365 return _internal_cm()
368def warm_tempdir(cwd: Any) -> None:
369 """Resolve `tempfile`'s temp directory before a task whose cwd has shifted.
371 `tempfile.gettempdir()` builds its candidate list by appending
372 `os.getcwd()` — eagerly, despite the stdlib comment calling it a last
373 resort — so the *first* `mkdtemp()` in a run reads the process cwd. The
374 getcwd guard would then attribute that read to the task, which never made
375 it: "task X reads the process cwd" about a line the author did not write.
377 Resolving it here rather than at `install()` is the whole point. The note
378 only fires when the task's directory differs from the process directory,
379 so a run started where the tasks file lives — the common case — has
380 nothing to prevent and should not pay ~3 ms to prevent it.
382 Called with `ctx.cwd` known and `ctx.in_task` still False, so `_managed_task`
383 reads unguarded and this resolution cannot trip the very note it exists to
384 avoid. Idempotent: `tempfile` caches its answer for the process, so the
385 flag only spares the repeat call.
386 """
387 global _tempdir_warmed
388 if _tempdir_warmed or cwd is None:
389 return
390 if str(cwd) == real_getcwd():
391 return # the answer would have been right anyway
392 _tempdir_warmed = True
393 import tempfile
395 # `run_task` sets `in_task` before the cwd is even resolved, so this
396 # resolution is guarded like a body's. It is not a body's: mark it as
397 # footman's own, or the warm earns the note it exists to prevent.
398 with internal():
399 tempfile.gettempdir()
402def _install_os_guards() -> None:
403 _guard_saved["chdir"] = os.chdir
404 _guard_saved["getcwd"] = os.getcwd
405 _guard_saved["putenv"] = getattr(os, "putenv", None)
406 _guard_saved["unsetenv"] = getattr(os, "unsetenv", None)
407 _guard_saved["fchdir"] = getattr(os, "fchdir", None)
408 _guard_saved["fork"] = getattr(os, "fork", None)
410 def _chdir_error(ctx: Any) -> RuntimeError:
411 return RuntimeError(
412 f"task {ctx.task or '?'} changes the process directory in a "
413 f"parallel task — the cwd belongs to no one there. The ladder: "
414 f"build paths from footman.cwd() (no chdir at all); or claim "
415 f"lanes=(cwd_lane,) — footman roots the real directory at the "
416 f"task's cwd for the hold; or mark the task serial to own the "
417 f"real globals and chdir freely."
418 )
420 orig_chdir: Callable[[Any], None] = _guard_saved["chdir"]
422 def chdir(path: Any) -> None:
423 ctx, guarded = _managed_task()
424 if guarded:
425 # A chdir to where the process already is changes nothing for
426 # anyone — the defensive restore pattern (pytest's wrap_session
427 # re-chdirs to its startpath on the way out). Only a real move
428 # is refused.
429 try:
430 same = os.path.realpath(path) == os.path.realpath(real_getcwd())
431 except (OSError, TypeError): # a dir fd, an unstatable path
432 same = False
433 if not same:
434 raise _chdir_error(ctx)
435 orig_chdir(path)
437 setattr(os, "chdir", chdir)
439 if _guard_saved["fchdir"] is not None:
440 orig_fchdir: Callable[[int], None] = _guard_saved["fchdir"]
442 def fchdir(fd: Any) -> None:
443 ctx, guarded = _managed_task()
444 if guarded:
445 raise _chdir_error(ctx)
446 orig_fchdir(fd)
448 setattr(os, "fchdir", fchdir)
450 orig_getcwd: Callable[[], str] = _guard_saved["getcwd"]
452 def getcwd() -> str:
453 here = orig_getcwd()
454 if _is_internal(): 454 ↛ 455line 454 didn't jump to line 455 because the condition on line 454 was never true
455 return here # footman resolving something on the task's behalf
456 ctx, guarded = _managed_task()
457 # Only when the answer is actually misleading. A task whose directory
458 # *is* the process directory — the common case, the runner started
459 # where the tasks file lives — gets a correct answer, and saying
460 # otherwise teaches people to ignore notes. The sibling chdir guard
461 # learned the same lesson: a move to where the process already is
462 # changes nothing for anyone, so it stopped being refused. A string
463 # compare against the already-resolved ctx.cwd, because getcwd can be
464 # polled in a loop where chdir can afford a realpath.
465 if guarded and ctx.cwd is not None and str(ctx.cwd) != here:
466 _note(
467 "getcwd",
468 f"task {ctx.task or '?'} reads the process cwd, which is not "
469 f"this task's directory ({ctx.cwd}) — a parallel run never "
470 f"chdirs. cwd() is the directory you want.",
471 )
472 return here
474 setattr(os, "getcwd", getcwd)
476 def _env_bypass_error(name: str) -> RuntimeError:
477 return RuntimeError(
478 f"os.{name} bypasses env scoping even in plain Python (it never "
479 f"updates os.environ) — assign through os.environ (scoped to "
480 f"this task), or pass env= to the call."
481 )
483 if _guard_saved["putenv"] is not None: 483 ↛ 494line 483 didn't jump to line 494 because the condition on line 483 was always true
484 orig_putenv = _guard_saved["putenv"]
486 def putenv(name: str, value: str) -> None:
487 _, guarded = _managed_task()
488 if guarded:
489 raise _env_bypass_error("putenv")
490 orig_putenv(name, value)
492 setattr(os, "putenv", putenv)
494 if _guard_saved["unsetenv"] is not None: 494 ↛ 505line 494 didn't jump to line 505 because the condition on line 494 was always true
495 orig_unsetenv = _guard_saved["unsetenv"]
497 def unsetenv(name: str) -> None:
498 _, guarded = _managed_task()
499 if guarded: 499 ↛ 500line 499 didn't jump to line 500 because the condition on line 499 was never true
500 raise _env_bypass_error("unsetenv")
501 orig_unsetenv(name)
503 setattr(os, "unsetenv", unsetenv)
505 if _guard_saved["fork"] is not None:
506 orig_fork: Callable[[], int] = _guard_saved["fork"]
508 def fork() -> int:
509 ctx, guarded = _managed_task()
510 if guarded: 510 ↛ 518line 510 didn't jump to line 518 because the condition on line 510 was always true
511 _note(
512 "fork",
513 f"task {ctx.task or '?'} forks — forking a threaded "
514 f"process is unsafe (the child can inherit locks "
515 f"mid-hold). Prefer run()/subprocess, or mark the task "
516 f"serial.",
517 )
518 return orig_fork()
520 setattr(os, "fork", fork)
523def _restore_os_guards() -> None:
524 if not _guard_saved: 524 ↛ 525line 524 didn't jump to line 525 because the condition on line 524 was never true
525 return
526 setattr(os, "chdir", _guard_saved["chdir"])
527 setattr(os, "getcwd", _guard_saved["getcwd"])
528 if _guard_saved["fchdir"] is not None:
529 setattr(os, "fchdir", _guard_saved["fchdir"])
530 if _guard_saved["putenv"] is not None: 530 ↛ 532line 530 didn't jump to line 532 because the condition on line 530 was always true
531 setattr(os, "putenv", _guard_saved["putenv"])
532 if _guard_saved["unsetenv"] is not None: 532 ↛ 534line 532 didn't jump to line 534 because the condition on line 532 was always true
533 setattr(os, "unsetenv", _guard_saved["unsetenv"])
534 if _guard_saved["fork"] is not None:
535 setattr(os, "fork", _guard_saved["fork"])
536 _guard_saved.clear()
539# --- multiprocessing detection ------------------------------------------------
541_mp_saved: Any = None
544def _install_multiprocessing() -> None:
545 global _mp_saved
546 if "multiprocessing.process" not in sys.modules: 546 ↛ 550line 546 didn't jump to line 550 because the condition on line 546 was never true
547 # Nothing has imported multiprocessing, so no task can have spawned a
548 # worker for the note to describe. Importing it here purely to install
549 # a diagnostic costs ~2.5 ms of every run's startup.
550 return
551 try:
552 from multiprocessing import process as mp_process
553 except Exception: # a stripped-down build without multiprocessing
554 return
555 _mp_saved = mp_process.BaseProcess.start
556 orig: Callable[[Any], None] = _mp_saved
558 def start(self: Any) -> None:
559 ctx, guarded = _managed_task()
560 if guarded: 560 ↛ 569line 560 didn't jump to line 569 because the condition on line 560 was always true
561 _note(
562 "mp-start",
563 f"task {ctx.task or '?'} spawns worker processes in-process "
564 f"— they inherit the real environment (run-wide state like "
565 f"colour is published there), but not the task's overlay. A "
566 f"tool that parallelises itself loses little in the serial "
567 f"lane: mark the task serial.",
568 )
569 return orig(self)
571 setattr(mp_process.BaseProcess, "start", start)
574def _restore_multiprocessing() -> None:
575 global _mp_saved
576 if _mp_saved is not None: 576 ↛ exitline 576 didn't return from function '_restore_multiprocessing' because the condition on line 576 was always true
577 from multiprocessing import process as mp_process
579 setattr(mp_process.BaseProcess, "start", _mp_saved)
580 _mp_saved = None
583# --- the argv router ----------------------------------------------------------
585_argv_saved: Any = None
586_argv_ctx: ContextVar[list[str] | None] = ContextVar("footman_argv", default=None)
589class _ArgvProxy(list[str]):
590 """The run's `sys.argv` — a real list underneath, with every Python-level
591 operation consulting the current call's override first.
593 This is what lets a legacy zero-argument `main()` run in parallel: each
594 in-process call sees its *own* argv through the same `sys.argv` object,
595 no lock, no global patch. A C extension reading the list storage directly
596 sees the base (the process's real argv) and degrades to the old
597 behaviour; direct reassignment (`sys.argv = […]`) replaces the proxy and
598 stays process-global — both documented edges, neither a crash."""
600 def _ov(self) -> list[str] | None:
601 return _argv_ctx.get()
603 def __getitem__(self, i: Any) -> Any:
604 ov = self._ov()
605 return ov[i] if ov is not None else super().__getitem__(i)
607 def __setitem__(self, i: Any, v: Any) -> None:
608 ov = self._ov()
609 if ov is not None: 609 ↛ 612line 609 didn't jump to line 612 because the condition on line 609 was always true
610 ov[i] = v
611 else:
612 super().__setitem__(i, v)
614 def __delitem__(self, i: Any) -> None:
615 ov = self._ov()
616 if ov is not None: 616 ↛ 619line 616 didn't jump to line 619 because the condition on line 616 was always true
617 del ov[i]
618 else:
619 super().__delitem__(i)
621 def __iter__(self) -> Any:
622 ov = self._ov()
623 return iter(ov) if ov is not None else super().__iter__()
625 def __len__(self) -> int:
626 ov = self._ov()
627 return len(ov) if ov is not None else super().__len__()
629 def __contains__(self, x: Any) -> bool:
630 ov = self._ov()
631 return x in ov if ov is not None else super().__contains__(x)
633 def __repr__(self) -> str:
634 ov = self._ov()
635 return repr(ov) if ov is not None else super().__repr__()
637 def __eq__(self, other: Any) -> bool:
638 ov = self._ov()
639 return (ov == other) if ov is not None else super().__eq__(other)
641 __hash__ = None # lists are unhashable
643 def __add__(self, other: Any) -> Any:
644 ov = self._ov()
645 return (ov + other) if ov is not None else list(self) + other
647 def append(self, v: Any) -> None:
648 ov = self._ov()
649 ov.append(v) if ov is not None else super().append(v)
651 def extend(self, it: Any) -> None:
652 ov = self._ov()
653 ov.extend(it) if ov is not None else super().extend(it)
655 def insert(self, i: Any, v: Any) -> None:
656 ov = self._ov()
657 ov.insert(i, v) if ov is not None else super().insert(i, v)
659 def pop(self, i: Any = -1) -> Any:
660 ov = self._ov()
661 return ov.pop(i) if ov is not None else super().pop(i)
663 def remove(self, v: Any) -> None:
664 ov = self._ov()
665 ov.remove(v) if ov is not None else super().remove(v)
667 def clear(self) -> None:
668 ov = self._ov()
669 ov.clear() if ov is not None else super().clear()
671 def index(self, *a: Any) -> int:
672 ov = self._ov()
673 return ov.index(*a) if ov is not None else super().index(*a)
675 def count(self, v: Any) -> int:
676 ov = self._ov()
677 return ov.count(v) if ov is not None else super().count(v)
679 def copy(self) -> list[str]:
680 ov = self._ov()
681 return list(ov) if ov is not None else list(self)
683 # In-place mutators. Unoverridden these fall through to `list`, which
684 # edits the proxy's own base storage — so the change vanishes from the
685 # caller (reads consult the override) *and* leaks into every call that
686 # has none. A legacy `main()` appending a default (`sys.argv += [...]`)
687 # is the realistic one; the rest are here so the set has no holes.
688 def __iadd__(self, other: Any) -> Any:
689 ov = self._ov()
690 if ov is None: 690 ↛ 691line 690 didn't jump to line 691 because the condition on line 690 was never true
691 return super().__iadd__(other)
692 ov.extend(other)
693 return self
695 def __imul__(self, n: Any) -> Any:
696 ov = self._ov()
697 if ov is None:
698 return super().__imul__(n)
699 ov *= n
700 return self
702 def __mul__(self, n: Any) -> Any:
703 ov = self._ov()
704 return (ov * n) if ov is not None else list(self) * n
706 __rmul__ = __mul__
708 def __reversed__(self) -> Any:
709 ov = self._ov()
710 return reversed(ov) if ov is not None else super().__reversed__()
712 def sort(self, **kw: Any) -> None:
713 ov = self._ov()
714 ov.sort(**kw) if ov is not None else super().sort(**kw)
716 def reverse(self) -> None:
717 ov = self._ov()
718 ov.reverse() if ov is not None else super().reverse()
721def argv_override(args: list[str]) -> Any:
722 """A context manager giving this thread its own `sys.argv` view for the
723 block — served by the argv router, lock-free. toolroom's bridge wraps a
724 legacy zero-argument `main()` in one, so those calls parallelise like
725 their argument-accepting siblings."""
726 import contextlib
728 @contextlib.contextmanager
729 def _cm() -> Any:
730 token = _argv_ctx.set(list(args))
731 try:
732 yield
733 finally:
734 _argv_ctx.reset(token)
736 return _cm()
739def _install_argv() -> None:
740 import sys as _sys
742 global _argv_saved
743 _argv_saved = _sys.argv
744 _sys.argv = _ArgvProxy(_argv_saved)
747def _restore_argv() -> None:
748 import sys as _sys
750 global _argv_saved
751 if _argv_saved is not None: 751 ↛ exitline 751 didn't return from function '_restore_argv' because the condition on line 751 was always true
752 _sys.argv = _argv_saved
753 _argv_saved = None
756# --- the stdin router ---------------------------------------------------------
758_stdin_saved: Any = None
761class _GuardedStdin:
762 """The parallel regime's stdin — the fourth global, guarded not served.
764 A read from a plain parallel task body is a taught error (one terminal,
765 consumed rather than mutated: it belongs to no one there); the
766 framework's own boundary prompts run outside task bodies, an
767 `interactive=True` or serial task owns the terminal legitimately, and
768 everything outside a run passes through untouched. `input()` is caught
769 too: with `sys.stdin` replaced, Python falls back to
770 `sys.stdin.readline()` instead of the C readline path.
771 """
773 def __init__(self, real: Any) -> None:
774 self._real = real
776 def _guard(self) -> None:
777 from footman.context import current
779 ctx = current()
780 if _installs and ctx.in_task and not ctx.interactive and not ctx.serial_active:
781 raise RuntimeError(
782 f"task {ctx.task or '?'} reads stdin in a parallel task — "
783 f"declare the value with ask(), or mark the task "
784 f"interactive=True to own the terminal."
785 )
787 def read(self, *a: Any) -> Any:
788 self._guard()
789 return self._real.read(*a)
791 def readline(self, *a: Any) -> Any:
792 self._guard()
793 return self._real.readline(*a)
795 def readlines(self, *a: Any) -> Any:
796 self._guard()
797 return self._real.readlines(*a)
799 def __iter__(self) -> Any:
800 self._guard()
801 return iter(self._real)
803 def __getattr__(self, name: str) -> Any: # fileno, isatty, encoding, …
804 return getattr(self._real, name)
807def _install_stdin() -> None:
808 import sys as _sys
810 global _stdin_saved
811 _stdin_saved = _sys.stdin
812 _sys.stdin = _GuardedStdin(_stdin_saved)
815def _restore_stdin() -> None:
816 import sys as _sys
818 global _stdin_saved
819 if _stdin_saved is not None: 819 ↛ exitline 819 didn't return from function '_restore_stdin' because the condition on line 819 was always true
820 _sys.stdin = _stdin_saved
821 _stdin_saved = None
824# --- the arbiter lanes --------------------------------------------------------
825#
826# Serialisation in the new regime is *declared* (serial= / exclusive=) and
827# acquired at task boundaries, where it can be scheduled instead of contended
828# for. The serial lane holds at most one owner and overlaps the parallel
829# pool; exclusive drains the world. A parent parked in a pool wait on its own
830# children is exempt from the drain (it is blocked in footman code and cannot
831# touch globals), and a child of a lane holder inherits the lane through its
832# context (`serial_active`), bypassing every bar — a lineage extends a hold,
833# it never contends with it.
835_arb_cv = threading.Condition(threading.Lock())
836_running = 0 # task bodies in flight (scheduler nodes + parallel() children)
837_parked = 0 # of those, parked waiting on their own children
838_serial_holder: str | None = None
839_excl_holder: str | None = None
840_console_holder: str | None = None
841_excl_waiting = 0
844class Lane:
845 """A serialised claim on one named resource: one holder at a time,
846 contending only with claimants of the SAME lane — unrelated work runs
847 untouched (nothing drains; that is `exclusive=`, a different regime).
849 Made by `footman.lane()` and claimed by handing the handle to a task's
850 or a step maker's `lanes=`: the import system is the registry, so a
851 misspelt lane is an undefined name, and sharing a resource across
852 modules is importing its binding. footman itself ships exactly two —
853 `cwd_lane` and `console_lane` — built with the same call.
854 """
856 __slots__ = ("name", "reason")
858 name: str
859 """The resource's one name — what the live line shows a wait on."""
860 reason: str | None
861 """Optional documentation on the declaration."""
863 def __init__(self, name: str, reason: str | None) -> None:
864 self.name = name
865 self.reason = reason
867 def __repr__(self) -> str:
868 return f"<lane {self.name}>"
871_lane_sites: dict[str, str] = {} # lane name -> the declaration site
874def make_lane(name: str, *, reason: str | None = None) -> Lane:
875 """Declare a named resource to serialise on. One binding per resource:
876 re-declaring a taken name is a refusal naming both sites — reuse is
877 spelled by importing the handle, never by re-declaring. (The same
878 site re-executing — a module re-imported — is the same declaration.)
879 """
880 # `sys._getframe`, not `inspect.stack()`: the site is one filename and one
881 # line number, and `inspect.stack()` builds a whole FrameInfo for every
882 # frame below — reading source files to do it. Measured at 2.1 ms a call
883 # from a tasks file, against 1.1 µs here, and footman declares two lanes
884 # at import time, so it was ~6 ms on the execution path before anything
885 # ran. Same answer, two thousand times cheaper.
886 frame = sys._getframe(1)
887 site = f"{frame.f_code.co_filename}:{frame.f_lineno}"
888 taken = _lane_sites.get(name)
889 if taken is not None and taken != site:
890 raise ValueError(
891 f"lane {name!r} is already declared at {taken} — one binding per "
892 f"resource. Share it by importing that handle; a second "
893 f"declaration would make two lanes that never contend."
894 )
895 _lane_sites[name] = site
896 return Lane(name, reason)
899_named_holders: dict[str, str] = {} # lane name -> holder, under _arb_cv
902def named_lanes(lanes: tuple[Lane, ...], name: str = "") -> Any:
903 """Hold *lanes* around one piece of work — a step's claims.
905 Granted atomically in a single predicate under the arbiter's one
906 condition variable (all lanes at once, no partial holds, so
907 hold-and-wait between lanes cannot be spelled), and released at the
908 one place the hold ends.
909 """
910 import contextlib
912 @contextlib.contextmanager
913 def _hold() -> Any:
914 global _parked
915 if not lanes:
916 yield []
917 return
918 wanted = {ln.name for ln in lanes}
919 stalled = False
920 # The lineage exemption `lane()` already applies, applied here too: a
921 # serial/exclusive task *is* the holder of the lane this claim would
922 # wait for, so waiting on it is waiting for itself. The hold is
923 # already total — it conflicts with every named lane — so a step
924 # inside it needs no further grant, and asking for one deadlocks the
925 # run deterministically.
926 from footman.context import current
928 mine = bool(getattr(current(), "serial_active", False))
929 with _arb_cv:
930 # Inside the lock on purpose: the clock must cover only the
931 # predicate wait. Started outside, a claim granted on arrival
932 # would book the mutex acquisition as a lane wait — a phantom
933 # row whenever the arbiter is briefly busy.
934 entered = time.monotonic()
935 # Parked across the wait, the same exemption a pool wait takes:
936 # this body is blocked in footman's own code and can touch nothing,
937 # so it must not hold the exclusive drain open. Without it the two
938 # predicates close a circle — the drain waits for every body to
939 # finish, and this claim waits for the queued drain — and the run
940 # hangs with nothing shared between the two tasks. Counted here
941 # rather than through `parked()`, which takes the arbiter lock this
942 # frame already holds.
943 #
944 # `_excl_waiting` stays in the predicate below: a queued exclusive
945 # must still block *new* claims, or a stream of them starves it.
946 _parked += 1
947 _arb_cv.notify_all()
948 try:
949 while any(w in _named_holders for w in wanted) or (
950 not mine
951 and (
952 _serial_holder is not None
953 or _excl_holder is not None
954 or _excl_waiting
955 )
956 ):
957 stalled = True
958 busy = next((w for w in wanted if w in _named_holders), None)
959 _wait_note(
960 name,
961 busy or "exclusive",
962 f"the {busy} lane" if busy else "the exclusive drain",
963 _named_holders.get(busy or "")
964 or _serial_holder
965 or _excl_holder,
966 )
967 finally:
968 _parked -= 1
969 for w in wanted:
970 _named_holders[w] = name or "?"
971 try:
972 yield _waited(",".join(sorted(wanted)), entered) if stalled else []
973 finally:
974 with _arb_cv:
975 for w in wanted:
976 _named_holders.pop(w, None)
977 _arb_cv.notify_all()
979 return _hold()
982def lane(
983 policy: str | None,
984 name: str = "",
985 inherited: bool = False,
986 console: bool = False,
987 named: tuple[Lane, ...] = (),
988) -> Any:
989 """A context manager holding *policy*'s lane around one task body.
991 `None` is the parallel regime: it only counts the body for the drain
992 and yields to a waiting exclusive first. `console=True` additionally
993 claims the terminal (an `interactive=True` body): one owner at a time,
994 granted **atomically with the policy lane** in a single predicate, so a
995 partial hold can never chain into hold-and-wait between lanes.
996 `inherited` marks a lineage child of a lane holder: it bypasses every
997 bar (the holder is waiting on it) and only counts.
998 """
999 import contextlib
1001 @contextlib.contextmanager
1002 def _lane() -> Any:
1003 global _running, _serial_holder, _excl_holder, _excl_waiting, _console_holder
1004 if not _installs:
1005 yield []
1006 return
1007 wanted = {ln.name for ln in named}
1008 if inherited:
1009 label = "" # a lineage bypasses every bar, so it never waits
1010 elif policy in ("serial", "exclusive"):
1011 label = policy
1012 elif wanted:
1013 label = ",".join(sorted(wanted))
1014 else:
1015 label = "console" if console else "exclusive"
1016 stalled = False
1017 with _arb_cv:
1018 # Inside the lock on purpose: the clock must cover only the
1019 # predicate wait, or a claim granted on arrival books the mutex
1020 # acquisition as a lane wait (a phantom row on a busy arbiter).
1021 entered = time.monotonic()
1022 if inherited:
1023 pass # a lineage extends every hold, the console included
1024 elif policy == "serial":
1025 # serial= IS the all-lanes claim: it conflicts with every
1026 # named holder, and every named claim waits on it.
1027 while (
1028 _serial_holder is not None
1029 or _excl_holder is not None
1030 or _excl_waiting
1031 or _named_holders
1032 or (console and _console_holder is not None)
1033 ):
1034 stalled = True
1035 _wait_note(
1036 name,
1037 "serial",
1038 "the serial lane",
1039 _serial_holder or _excl_holder,
1040 )
1041 _serial_holder = name or "?"
1042 elif policy == "exclusive":
1043 _excl_waiting += 1
1044 try:
1045 while (
1046 _excl_holder is not None
1047 or _serial_holder is not None
1048 or (_running - _parked) > 0
1049 or _named_holders
1050 or (console and _console_holder is not None)
1051 ):
1052 stalled = True
1053 _wait_note(
1054 name, "exclusive", "the exclusive drain", _serial_holder
1055 )
1056 finally:
1057 _excl_waiting -= 1
1058 _excl_holder = name or "?"
1059 else:
1060 while (
1061 _excl_holder is not None
1062 or _excl_waiting
1063 or (console and _console_holder is not None)
1064 or any(w in _named_holders for w in wanted)
1065 ):
1066 stalled = True
1067 busy = next((w for w in wanted if w in _named_holders), None)
1068 what = (
1069 f"the {busy} lane"
1070 if busy
1071 else ("the console" if console else "the exclusive drain")
1072 )
1073 lane = busy or ("console" if console else "exclusive")
1074 _wait_note(name, lane, what, _console_holder or _excl_holder)
1075 if console and not inherited:
1076 _console_holder = name or "?"
1077 for w in wanted:
1078 _named_holders[w] = name or "?"
1079 _running += 1
1080 claimed_console = console and not inherited
1081 if claimed_console:
1082 _suspend_status(True) # outside the cv: lock order is arb → status
1083 try:
1084 yield _waited(label, entered) if stalled and label else []
1085 finally:
1086 with _arb_cv:
1087 _running -= 1
1088 if not inherited:
1089 if policy == "serial":
1090 _serial_holder = None
1091 elif policy == "exclusive":
1092 _excl_holder = None
1093 if console:
1094 _console_holder = None
1095 for w in wanted:
1096 _named_holders.pop(w, None)
1097 _arb_cv.notify_all()
1098 if claimed_console:
1099 _suspend_status(False)
1101 return _lane()
1104def _suspend_status(on: bool) -> None:
1105 """Pause/resume the live status line around console ownership — its
1106 repaints and a wizard's prompt would fight for the one terminal."""
1107 from footman.context import active_status
1109 status = active_status()
1110 if status is None:
1111 return
1112 (status.suspend if on else status.resume)()
1115def console_gate() -> Any:
1116 """Hold a real-terminal write until no console owner is active.
1118 A captured sibling finishing mid-wizard must not splat its buffered
1119 block over the wizard's screen: the flush queues here and lands when
1120 the terminal frees. Best-effort by design — a wizard *starting* between
1121 the gate and the write is the same race as one starting just after it,
1122 and the write is a single atomic block either way."""
1123 import contextlib
1125 @contextlib.contextmanager
1126 def _gate() -> Any:
1127 if not _installs:
1128 yield
1129 return
1130 with _arb_cv:
1131 while _console_holder is not None:
1132 _arb_cv.wait(timeout=1.0)
1133 yield
1135 return _gate()
1138def _wait_note(name: str, lane: str, what: str, holder: str | None) -> None:
1139 """One bounded wait step, with a one-time visibility note — a lane wait
1140 must never be a silent hang. The *lane* is the kind's instance (what
1141 is being waited for); the waiting task is the dedup's other half
1142 already, so it never rides the kind."""
1143 if not _arb_cv.wait(timeout=2.0): 1143 ↛ 1144line 1143 didn't jump to line 1144 because the condition on line 1143 was never true
1144 held = f" (held by {holder})" if holder else ""
1145 _note(f"lane-wait:{lane}", f"task {name or '?'} waiting for {what}{held}")
1148def _waited(label: str, entered: float) -> list[tuple[str, float]]:
1149 """What this claim paid at the bar, for the run report.
1151 Called only after the claim's predicate genuinely stalled; the clock
1152 started inside the arbiter lock, so mutex acquisition never counts. One
1153 `(label, seconds)` row, unless the stall resolved inside a millisecond —
1154 that is arrival, not serialisation. The claim's own label, not the
1155 blocker-of-the-moment: a wait can pass through several holders, and what
1156 the report answers is "what serialised *this*"."""
1157 waited = time.monotonic() - entered
1158 return [(label, waited)] if waited >= 0.001 else []
1161def parked() -> Any:
1162 """Mark the current body parked (a pool wait on its own children): it is
1163 blocked in footman code and cannot touch globals, so the exclusive drain
1164 exempts it — this is the ancestry exemption."""
1165 import contextlib
1167 @contextlib.contextmanager
1168 def _parked_cm() -> Any:
1169 global _parked
1170 if not _installs:
1171 yield
1172 return
1173 with _arb_cv:
1174 _parked += 1
1175 _arb_cv.notify_all()
1176 try:
1177 yield
1178 finally:
1179 with _arb_cv:
1180 _parked -= 1
1182 return _parked_cm()
1185_gc_deferred = False
1188def defer_gc() -> None:
1189 """Stop collecting until the run reaches task bodies.
1191 Startup allocates a great deal and discards almost none of it: the
1192 framework's own modules, then the user's tasks file. Left alone, CPython
1193 runs about eleven collections getting there, each one walking objects that
1194 are all still live. Only the console-script entry calls this, because only
1195 it owns the whole process; `Runner` drives a run inside somebody else's
1196 process and must not touch a global switch.
1197 """
1198 global _gc_deferred
1199 import gc
1201 gc.disable()
1202 _gc_deferred = True
1205def _resume_gc() -> None:
1206 """Hand startup's objects to the permanent generation, then collect again.
1208 Paired with `defer_gc`, and a no-op without it. The freeze is what makes
1209 re-enabling cheap: everything allocated up to here is live and will stay
1210 live for the few milliseconds this process has left, so it should never be
1211 traversed again — while a task body that makes real garbage still gets a
1212 real collector. Forcing a collection here instead measured *slower* than
1213 plain: the traversal costs more than the garbage is worth.
1214 """
1215 global _gc_deferred
1216 if not _gc_deferred:
1217 return
1218 import gc
1220 gc.freeze()
1221 gc.enable()
1222 _gc_deferred = False
1225def install() -> None:
1226 """Arm the routers for a run. Refcounted; the first install pins the
1227 environment snapshot (so anything published at the run boundary —
1228 colour, say — is in it)."""
1229 global _installs
1230 with _lock:
1231 _installs += 1
1232 if _installs > 1:
1233 return
1234 # The last thing before task bodies run, and the reason `install` is
1235 # where this lives: everything the run needed to get here is loaded.
1236 _resume_gc()
1237 _snapshot.clear()
1238 _snapshot.update(os.environ)
1239 from footman import _notes
1241 _notes.reset() # every note may teach once more
1242 _install_environ()
1243 _install_popen()
1244 _install_os_guards()
1245 _install_multiprocessing()
1246 _install_stdin()
1247 _install_argv()
1250def uninstall() -> None:
1251 """Disarm after a run; the last uninstall restores the originals."""
1252 global _installs
1253 with _lock:
1254 if _installs == 0: 1254 ↛ 1255line 1254 didn't jump to line 1255 because the condition on line 1254 was never true
1255 return
1256 _installs -= 1
1257 if _installs > 0:
1258 return
1259 _restore_argv()
1260 _restore_stdin()
1261 _restore_multiprocessing()
1262 _restore_os_guards()
1263 _restore_popen()
1264 _restore_environ()
1265 _snapshot.clear()
1268# The two lanes footman itself ships — the only core lanes there will ever
1269# be, declared with the very call plugins and tasks files use. cwd is
1270# opt-in: claiming it is knowingly giving up some parallelism, and the
1271# import at the top of the file says so where reviewers read. The console
1272# is claimed implicitly by `interactive=` and explicitly here.
1273cwd_lane: Lane = make_lane("cwd", reason="the one real working directory")
1274console_lane: Lane = make_lane("console", reason="the one terminal")