Coverage for src/footman/_fetch.py: 97%
222 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-09-06 02:59 +0000
1"""Download files into footman's cache — `fetch()`.
3Build tasks fetch things: a toolchain tarball, a schema, a fixture. Doing
4it well means caching by URL, revalidating instead of re-downloading,
5verifying what arrived, and reporting progress — and doing it *here*
6means it composes with everything already built:
8- the cached copy lives under `footman_cache_dir()`, so
9 `FOOTMAN_CACHE_DIR` relocates it and the cache collector tends it;
10- every fetch records a `Result`, so `--dry-run` prints without
11 downloading, `recording()` asserts on it in tests, `--json` carries
12 it, and the step lines show it in the same aligned grid as `run()`;
13- byte counts feed `progress()`, so a download drives the live bar.
15**Backends.** The default is stdlib `urllib` — always present, zero
16dependencies, deterministic, and the only backend that can report bytes
17as they arrive. `curl` (shipped in Windows' System32 since build 17063,
18and on every POSIX box) is the escape hatch for corporate proxies and
19TLS stores that Python's defaults can't see; `httpx` and `requests` are
20used only when explicitly named. Choose per call, or set
21`[fetch] backend` in any config file — a machine behind a proxy sets it
22once in `~/.config/footman/config.toml` and every project follows.
24Which one you name changes the socket, and nothing else. All four
25revalidate with the same headers, refuse a body that arrived short, and
26land the finished file with the same rename — a backend that behaved its
27own way would make the choice a behaviour change rather than a plumbing
28one, and the conformance table in `tests/test_fetch.py` runs one set of
29scenarios against every backend installed to keep it that way.
31Deliberately *not* automatic: a fetch that silently picks a different
32engine depending on what happens to be importable would change its TLS
33trust store and proxy semantics when an unrelated dependency appears.
34`backend = "auto"` exists for people who want that, spelled out as a
35choice rather than a surprise.
37This is for build artifacts, not a general HTTP client. Anything exotic
38belongs in a real curl call — `run(["curl", ...])`, or toolroom's typed
39`curl(...)` handle.
40"""
42from __future__ import annotations
44import contextlib
45import hashlib
46import http.client
47import json
48import os
49import shutil
50import tempfile
51import time
52import urllib.error
53import urllib.request
54from pathlib import Path
55from typing import Any
57from footman import _paths, context
59BACKENDS = ("urllib", "curl", "httpx", "requests", "auto")
60_AUTO_ORDER = ("httpx", "requests", "urllib", "curl")
61CHUNK = 64 * 1024
64class FetchError(Exception):
65 """A download failed, or arrived wrong (checksum, missing backend)."""
68def cache_dir() -> Path:
69 """Where fetched files live: a `fetch/` room in footman's own cache."""
70 return _paths.footman_cache_dir() / "fetch"
73def _key(url: str) -> str:
74 return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
77def _manifest_path(url: str) -> Path:
78 """The URL's manifest: a small JSON that names its own data file.
80 The layout's one moving part. The data file is content-addressed
81 (`<key>-<digest16>.bin`) and immutable once published; a fresh download
82 lands under a fresh name and only this manifest is replaced — one
83 uncontended swap of a small file no reader holds open. A path handed to
84 a caller therefore never changes underneath them, the validators in the
85 manifest describe exactly the bytes they sit beside, and the Windows
86 replace-under-reader failure has nothing left to happen to.
87 """
88 return cache_dir() / f"{_key(url)}.json"
91def _data_name(url: str, digest: str) -> str:
92 return f"{_key(url)}-{digest[:16]}.bin"
95def _cached_body(url: str, meta: dict[str, Any]) -> Path | None:
96 """The data file *meta* names, when it is really there."""
97 name = meta.get("data")
98 if not isinstance(name, str) or not name.startswith(_key(url)):
99 return None
100 body = cache_dir() / name
101 return body if body.is_file() else None
104def _load_meta(path: Path) -> dict[str, Any]:
105 try:
106 data = json.loads(path.read_text("utf-8"))
107 except (OSError, ValueError):
108 return {}
109 return data if isinstance(data, dict) else {}
112def _digest(path: Path) -> str:
113 sha = hashlib.sha256()
114 with open(path, "rb") as fh:
115 for block in iter(lambda: fh.read(CHUNK), b""):
116 sha.update(block)
117 return sha.hexdigest()
120def _resolve_backend(name: str) -> str:
121 """The backend to use, refusing a named-but-missing one out loud."""
122 if name == "auto":
123 for candidate in _AUTO_ORDER: 123 ↛ 126line 123 didn't jump to line 126 because the loop on line 123 didn't complete
124 if _available(candidate):
125 return candidate
126 raise FetchError("fetch: no usable backend (not even urllib?)")
127 if name not in BACKENDS:
128 options = ", ".join(BACKENDS)
129 raise FetchError(f"fetch: unknown backend {name!r} — choose one of {options}")
130 if not _available(name):
131 if name == "curl":
132 raise FetchError("fetch: backend 'curl' is not on PATH")
133 raise FetchError(
134 f"fetch: backend {name!r} is not installed — `pip install {name}`, "
135 f"or leave [fetch] backend unset to use the stdlib"
136 )
137 return name
140def _available(name: str) -> bool:
141 if name == "urllib":
142 return True
143 if name == "curl":
144 return shutil.which("curl") is not None
145 import importlib.util
147 return importlib.util.find_spec(name) is not None
150def _scratch(near: Path, suffix: str = ".part") -> Path:
151 """A unique scratch file beside *near*, for a download in flight.
153 Beside, so the finished body lands on its cache path with a rename on
154 the same filesystem. Named `.part`, so the collector can sweep one
155 whose process died before it got that far.
156 """
157 fd, name = tempfile.mkstemp(dir=near.parent, prefix=f"{near.stem}.", suffix=suffix)
158 os.close(fd)
159 return Path(name)
162def _download(
163 backend: str, url: str, dest: Path, meta: dict[str, Any]
164) -> tuple[bool, dict[str, Any]]:
165 """Fetch *url* into *dest*; returns `(downloaded, validators)`.
167 Two separate facts that used to share one value: whether bytes moved
168 (what the receipt reports), and the validators to persist for the next
169 revalidation (what the manifest stores). A revalidation that comes back
170 `304` moves no bytes and keeps the validators it already had, which
171 under the shared value read as a fresh download of nothing.
173 *dest* is a scratch path, never the cache path — `fetch()` renames it
174 into place, so every backend inherits the same atomic landing."""
175 if backend == "curl":
176 return _download_curl(url, dest, meta)
177 if backend in ("httpx", "requests"):
178 return _download_lib(backend, url, dest, meta)
179 return _download_urllib(url, dest, meta)
182def _conditional_headers(meta: dict[str, Any]) -> dict[str, str]:
183 headers = {}
184 if etag := meta.get("etag"):
185 headers["If-None-Match"] = str(etag)
186 if modified := meta.get("last_modified"): 186 ↛ 187line 186 didn't jump to line 187 because the condition on line 186 was never true
187 headers["If-Modified-Since"] = str(modified)
188 return headers
191def _download_urllib(
192 url: str, dest: Path, meta: dict[str, Any]
193) -> tuple[bool, dict[str, Any]]:
194 request = urllib.request.Request(url, headers=_conditional_headers(meta))
195 try:
196 with urllib.request.urlopen(request) as response:
197 total = int(response.headers.get("Content-Length") or 0)
198 received = 0
199 with open(dest, "wb") as fh:
200 while chunk := response.read(CHUNK):
201 fh.write(chunk)
202 received += len(chunk)
203 if total:
204 context.progress(received, total)
205 if total: 205 ↛ 207line 205 didn't jump to line 207 because the condition on line 205 was always true
206 context.progress(0, 0) # done reporting: back to the estimate
207 if total and received != total:
208 # A short body reads as a small one to everyone upstream, and
209 # cached it is forever: the ETag off this same response goes
210 # in the manifest, the healthy origin answers 304 from then
211 # on, and the half file is served as a hit. CPython won't
212 # raise here itself — HTTPResponse.read's own comment says it
213 # "might break compatibility" — so the check is ours to make.
214 raise FetchError(
215 f"fetch: {url} — the body ended early: {received} of {total} bytes"
216 )
217 return True, {
218 "etag": response.headers.get("ETag"),
219 "last_modified": response.headers.get("Last-Modified"),
220 }
221 except urllib.error.HTTPError as exc:
222 if exc.code == 304: # not modified: the cached copy stands
223 return False, {}
224 raise FetchError(f"fetch: {url} — HTTP {exc.code} {exc.reason}") from exc
225 except urllib.error.URLError as exc:
226 raise FetchError(
227 f"fetch: {url} — {exc.reason}. If this machine needs the system "
228 f'curl (a corporate proxy or TLS store), set `backend = "curl"` '
229 f"under [fetch] — in this project, or once for every project in "
230 f"{_paths.footman_config_file()}"
231 ) from exc
232 except (OSError, http.client.HTTPException) as exc:
233 # A connection that dies mid-body, after the headers arrived. It has
234 # to read as a FetchError or the cached copy never gets its say.
235 raise FetchError(f"fetch: {url} — {exc}") from exc
238def _download_curl(
239 url: str, dest: Path, meta: dict[str, Any]
240) -> tuple[bool, dict[str, Any]]:
241 import subprocess
243 from footman import _globals
245 dump = _scratch(dest, suffix=".headers.part")
246 argv = ["curl", "-fsSL", "--retry", "2", "-D", str(dump), "-o", str(dest), url]
247 for header, value in _conditional_headers(meta).items():
248 argv += ["-H", f"{header}: {value}"]
249 # This spawn is footman working on the body's behalf, so the Popen
250 # injector must not attribute it to the task — the "prefer run()" note is
251 # teach-once, and spending it here swallows the note a real raw spawn in
252 # the same task would have earned. `internal()` stands the injector down;
253 # cwd and env are passed explicitly so nothing changes with it gone:
254 # every path on the command line is absolute (cwd is unobservable, pinned
255 # to the cache directory anyway), and the environment is snapshotted
256 # through the router *before* entering `internal()`, so a managed task's
257 # curl sees the same proxy variables the urllib backend reads in-process.
258 env = dict(os.environ)
259 try:
260 with _globals.internal():
261 done = subprocess.run(
262 argv, capture_output=True, text=True, cwd=dest.parent, env=env
263 )
264 if done.returncode != 0:
265 # Exit 18 lives here too: curl counts the body against
266 # Content-Length itself and calls a short one a failure.
267 raise FetchError(f"fetch: {url} — curl: {done.stderr.strip()}")
268 code, headers = _last_response(dump.read_text("utf-8", errors="replace"))
269 finally:
270 with contextlib.suppress(OSError):
271 dump.unlink()
272 if code == 304: # not modified: curl wrote nothing, the cached copy stands
273 return False, {}
274 return True, {
275 "etag": headers.get("etag"),
276 "last_modified": headers.get("last-modified"),
277 }
280def _last_response(dump: str) -> tuple[int, dict[str, str]]:
281 """The status and headers of the final response in a curl `-D` dump.
283 Redirects and retries each append their own block; only the last one
284 describes the bytes that landed in the file.
285 """
286 code = 0
287 headers: dict[str, str] = {}
288 for line in dump.splitlines():
289 if line.upper().startswith("HTTP/"):
290 parts = line.split()
291 code = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else 0
292 headers = {}
293 elif ":" in line:
294 name, _, value = line.partition(":")
295 headers[name.strip().lower()] = value.strip()
296 return code, headers
299def _download_lib(
300 name: str, url: str, dest: Path, meta: dict[str, Any]
301) -> tuple[bool, dict[str, Any]]:
302 import importlib
304 client = importlib.import_module(name)
305 # Each library's own error base, reached through the module object that is
306 # already in hand. Naming them costs no import up here — which is the one
307 # thing this file may not do — and `except Exception` was too much net: it
308 # dressed footman's own bugs as network failures and then served a stale
309 # cached copy on the strength of it. Both bases cover the transport
310 # failures that matter (httpx's RemoteProtocolError, requests'
311 # ChunkedEncodingError) and nothing else.
312 transport_error: type[BaseException] = (
313 client.HTTPError if name == "httpx" else client.exceptions.RequestException
314 )
315 try:
316 response = (
317 client.get(url, headers=_conditional_headers(meta), follow_redirects=True)
318 if name == "httpx"
319 else client.get(
320 url, headers=_conditional_headers(meta), allow_redirects=True
321 )
322 )
323 if response.status_code == 304:
324 return False, {}
325 if response.status_code >= 400:
326 raise FetchError(f"fetch: {url} — HTTP {response.status_code}")
327 payload = response.content
328 except FetchError:
329 raise
330 except transport_error as exc:
331 # A dropped connection has to arrive as a FetchError, or the cached
332 # copy never gets its say.
333 raise FetchError(f"fetch: {url} — {type(exc).__name__}: {exc}") from exc
334 dest.write_bytes(payload)
335 return True, {
336 "etag": response.headers.get("ETag"),
337 "last_modified": response.headers.get("Last-Modified"),
338 }
341def fetch(
342 url: str,
343 *,
344 into: Path | str | None = None,
345 sha256: str = "",
346 backend: str = "",
347 refresh: bool = False,
348) -> Path:
349 """Download *url* (cached), returning the path to the local file.
351 A second call for the same URL revalidates with the server (ETag /
352 Last-Modified) rather than re-downloading; a `304 Not Modified`
353 costs one round trip and keeps "cached" honest. Pass *refresh* to
354 skip revalidation and fetch unconditionally.
356 ```python
357 @task
358 def deps():
359 "Fetch the toolchain."
360 archive = fetch(TOOLCHAIN_URL, sha256="9f86d0…")
361 run(["tar", "-xzf", archive, "-C", "vendor"])
362 ```
364 *into* copies the cached file to a path of your choosing (and
365 returns that path). *sha256* verifies what arrived and refuses a
366 mismatch — the way to make a build reproducible. *backend* overrides
367 the configured one for this call.
369 Under `--dry-run` nothing is downloaded: the step is recorded and
370 the would-be cache path returned, so a plan can be inspected safely.
371 """
372 ctx = context.current()
373 label = f"fetch {url}"
374 manifest = _manifest_path(url)
375 meta = _load_meta(manifest)
376 body = _cached_body(url, meta)
377 # `refresh` skips revalidation, not the cache: the download goes out
378 # unconditional, but a cached copy keeps its say as the fallback when
379 # that download dies — the documented offline behaviour.
380 validators = {} if refresh else meta
382 if ctx.dry_run:
383 ctx.steps.append(context.Result(0, command=label, raw=label))
384 if not ctx.quiet: 384 ↛ 388line 384 didn't jump to line 388 because the condition on line 384 was always true
385 print(f"$ {label}")
386 # The cached path when there is one; otherwise the key's nominal
387 # stem — a stable placeholder for the plan, never a real file.
388 nominal = body or cache_dir() / f"{_key(url)}.bin"
389 return Path(into) if into is not None else nominal
391 chosen = _resolve_backend(backend or _configured_backend(ctx))
392 started = time.perf_counter()
393 cache_dir().mkdir(parents=True, exist_ok=True)
394 incoming = _scratch(manifest)
395 try:
396 downloaded, fresh = _download(
397 chosen, url, incoming, validators if body is not None else {}
398 )
399 if downloaded:
400 # The data file is content-addressed and immutable: a fresh
401 # download lands under a fresh name, so no reader's path is
402 # ever replaced underneath them — two cold fetches of one URL
403 # publish the same name from the same bytes, and a changed
404 # upstream publishes a different one. Only the manifest moves.
405 digest = _digest(incoming)
406 final = cache_dir() / _data_name(url, digest)
407 if final.exists():
408 # Same content already published: the bytes are identical
409 # by construction, so the part is a duplicate — and not
410 # replacing means a Windows reader's open handle is safe.
411 incoming.unlink()
412 else:
413 os.replace(incoming, final)
414 body = final
415 _publish(
416 manifest,
417 {"url": url, "data": final.name, "sha256": digest, **(fresh or {})},
418 )
419 except FetchError:
420 if body is not None: # a cached copy beats a failed refresh
421 _touch(manifest, body)
422 _record(ctx, label, started, cached=True)
423 return _deliver(body, _destination(into, body), sha256, url)
424 raise
425 finally:
426 with contextlib.suppress(OSError):
427 incoming.unlink(missing_ok=True)
428 if body is None: 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true
429 raise FetchError(f"fetch: {url} — nothing downloaded and nothing cached")
430 if not downloaded:
431 if fresh: 431 ↛ 434line 431 didn't jump to line 434 because the condition on line 431 was never true
432 # A revalidation may hand back fresh validators for the same
433 # bytes (a rotated ETag): same data file, updated manifest.
434 _publish(manifest, {**meta, **fresh})
435 # A serve is a use: the collector's idle rule reads the manifest's
436 # mtime, and a 304 writes nothing of its own — untouched, a
437 # daily-fetched file would age out and force a pointless re-download.
438 _touch(manifest, body)
439 _record(ctx, label, started, cached=not downloaded)
440 return _deliver(body, _destination(into, body), sha256, url)
443def _destination(into: Path | str | None, body: Path) -> Path:
444 return Path(into) if into is not None else body
447def _publish(manifest: Path, data: dict[str, Any]) -> None:
448 """The manifest, replaced whole — the layout's one swap.
450 Written beside and moved on, so a concurrent reader sees the old
451 manifest or the new one, never a torn half; the file is small and no
452 reader holds it open, so the swap is uncontended on every platform.
453 """
454 scratch = _scratch(manifest, suffix=".json.part")
455 scratch.write_text(json.dumps(data), encoding="utf-8")
456 os.replace(scratch, manifest)
459def _touch(*paths: Path) -> None:
460 """Mark a cache serve as use, for the collector's idle rule."""
461 for path in paths:
462 with contextlib.suppress(OSError):
463 os.utime(path)
466def _configured_backend(ctx: context.Context) -> str:
467 """`[fetch] backend` from the config ladder, defaulting to urllib."""
468 return ctx.fetch_backend or "urllib"
471def _deliver(body: Path, destination: Path, sha256: str, url: str) -> Path:
472 if sha256:
473 actual = _digest(body)
474 if actual != sha256.lower():
475 raise FetchError(
476 f"fetch: {url} — sha256 mismatch\n expected {sha256.lower()}\n"
477 f" received {actual}"
478 )
479 if destination != body:
480 destination.parent.mkdir(parents=True, exist_ok=True)
481 shutil.copyfile(body, destination)
482 return destination
485def _record(ctx: context.Context, label: str, started: float, *, cached: bool) -> None:
486 """A fetch is a step: same grid, same --json entry, same recording()."""
487 note = "cached" if cached else ""
488 ctx.steps.append(
489 context.Result(
490 0,
491 command=label,
492 stdout=note,
493 duration=time.perf_counter() - started,
494 raw=label,
495 )
496 )