Coverage for src/footman/_gc.py: 97%

1"""Cache garbage collection — the detached child a run spawns at most daily. 

2 

3The cache is derived state top to bottom (completion manifests rebuild on 

4any execution-path run; timing history regrows), which is what makes 

5collecting it casually safe: the worst possible outcome of any deletion is 

6a rebuild. Two rules, in order: 

7 

81. **The directory is gone.** Manifests bake in the ``cwd`` they describe; 

9 if that path no longer exists, the pair (manifest + timing history) is 

10 leftovers from a deleted project — collected at any age. Only a file 

11 footman recognises as its own manifest is read this way; a task author is 

12 invited to keep state in ``cache_dir()``, and one of those files with a 

13 ``cwd`` key of its own is nobody's leftovers. 

142. **The pair is idle.** Untouched for `IDLE_DAYS`, nobody even TAB-completes 

15 there any more (background refreshes keep a visited manifest's mtime 

16 fresh) — collected. Manifests from before the ``cwd`` key, and anything 

17 in the directory footman did not write, rely on this rule alone. 

183. **The fetch room lives by reference, and only its clocks by age.** 

19 ``fetch/`` holds one small manifest per URL naming a content-addressed 

20 data file. The manifest is the clock: a cache serve touches it 

21 (`_fetch._touch`), so an idle one means nothing asked in `IDLE_DAYS` and 

22 it goes. A live manifest pins the data file it names at any age; a data 

23 file no manifest names is unreachable — fetch resolves only through 

24 manifests — so it is garbage immediately, however young. ``.part`` 

25 files (downloads in flight, referenced by nothing yet) keep the one 

26 remaining clock: a killed process's leftover ages out on `PART_DAYS`, 

27 while a live download keeps its own mtime fresh by writing. The old 

28 ``<key>.bin`` + ``.meta.json`` pairs age exactly as they always did. 

29 

30The invoking directory's own pair is never touched, and every failure is 

31silent — a concurrently-reading completion child on Windows may hold a file 

32open, and a collector must never be louder than what it collects. 

33""" 

34 

35from __future__ import annotations 

36 

37import contextlib 

38import json 

39import sys 

40import time 

41from pathlib import Path 

42 

43IDLE_DAYS = 90 

44PART_DAYS = 1 

45STAMP = "gc.stamp" 

46 

47 

48def collect(cache_dir: Path, skip_stem: str = "") -> int: 

49 """Apply the rules to *cache_dir*; returns the number of files removed. 

50 

51 *skip_stem* is the invoking directory's manifest stem (its hash) — that 

52 pair is in active use and never touched. 

53 """ 

54 now = time.time() 

55 removed = 0 

56 

57 def unlink(path: Path) -> None: 

58 nonlocal removed 

59 try: 

60 path.unlink() 

61 removed += 1 

62 except OSError: 

63 pass 

64 

65 for manifest_path in cache_dir.glob("*.json"): 

66 name = manifest_path.name 

67 if name.endswith(".times.json"): 

68 continue # handled with its manifest, or as an orphan below 

69 stem = name[: -len(".json")] 

70 if stem == skip_stem: 

71 continue 

72 times_path = cache_dir / f"{stem}.times.json" 

73 try: 

74 data = json.loads(manifest_path.read_text("utf-8")) 

75 except (OSError, ValueError): 

76 data = None 

77 cwd = _baked_cwd(data) 

78 if cwd and not Path(cwd).exists(): 

79 doomed = True # rule 1: leftovers of a deleted project 

80 else: 

81 doomed = _idle(now, manifest_path, times_path) # rule 2 

82 if doomed: 

83 unlink(manifest_path) 

84 unlink(times_path) 

85 

86 # Timing files whose manifest twin is already gone: age them alone. 

87 for times_path in cache_dir.glob("*.times.json"): 

88 stem = times_path.name[: -len(".times.json")] 

89 if stem == skip_stem or (cache_dir / f"{stem}.json").exists(): 

90 continue 

91 if _idle(now, times_path): 

92 unlink(times_path) 

93 

94 # Rule 3, the fetch room. The manifest is the clock and the reference: 

95 # an idle manifest goes (nobody asked in IDLE_DAYS — a serve touches 

96 # it), and a live one pins the data file it names. Completed data needs 

97 # no clock of its own — a data file no manifest names is unreachable 

98 # (fetch resolves only through manifests), so it is garbage the moment 

99 # it is orphaned, however young. Only a download in flight keeps a 

100 # clock, because nothing references it *yet*. 

101 fetch_room = cache_dir / "fetch" 

102 referenced: set[str] = set() 

103 for manifest in fetch_room.glob("*.json"): 

104 if manifest.name.endswith(".meta.json"): 

105 continue # the old layout's sidecar, aged with its pair below 

106 try: 

107 data = json.loads(manifest.read_text("utf-8")) 

108 except (OSError, ValueError): 

109 data = None 

110 data_name = data.get("data") if isinstance(data, dict) else None 

111 if _idle(now, manifest): 

112 unlink(manifest) 

113 elif isinstance(data_name, str): 113 ↛ 103line 113 didn't jump to line 103 because the condition on line 113 was always true

114 referenced.add(data_name) 

115 for body in fetch_room.glob("*.bin"): 

116 if "-" in body.stem: 

117 # New layout: content-addressed, reachable only via a manifest. 

118 if body.name not in referenced: 

119 unlink(body) 

120 continue 

121 # Old layout (`<key>.bin` + sidecar): the age rule it always had. 

122 sidecar = fetch_room / f"{body.stem}.meta.json" 

123 if _idle(now, body, sidecar): 

124 unlink(body) 

125 unlink(sidecar) 

126 for sidecar in fetch_room.glob("*.meta.json"): 

127 stem = sidecar.name[: -len(".meta.json")] 

128 if (fetch_room / f"{stem}.bin").exists(): 

129 continue 

130 if _idle(now, sidecar): 130 ↛ 126line 130 didn't jump to line 126 because the condition on line 130 was always true

131 unlink(sidecar) 

132 # A download in flight keeps its own mtime fresh; one that stopped 

133 # writing a day ago belongs to a process that is not coming back. 

134 for part in fetch_room.glob("*.part"): 

135 if _idle(now, part, days=PART_DAYS): 

136 unlink(part) 

137 

138 return removed 

139 

140 

141def _baked_cwd(data: object) -> str: 

142 """The directory *data* describes, or `""` if it is not one of ours. 

143 

144 Rule 1 reads meaning out of a file and no age protects what it condemns, 

145 so it may only speak for a file footman wrote. A task is invited to keep 

146 its own state in `cache_dir()`; one of those that happens to carry a `cwd` 

147 key of its own is judged by age alone, like an unparseable file. `tree` 

148 and `hash` have been in every manifest since the first schema. 

149 """ 

150 if not isinstance(data, dict) or "tree" not in data or "hash" not in data: 

151 return "" 

152 cwd = data.get("cwd") 

153 return cwd if isinstance(cwd, str) else "" 

154 

155 

156def _idle(now: float, *paths: Path, days: float = IDLE_DAYS) -> bool: 

157 """Whether the newest of *paths* is older than the idle window.""" 

158 newest = 0.0 

159 for path in paths: 

160 with contextlib.suppress(OSError): 

161 newest = max(newest, path.stat().st_mtime) 

162 return newest > 0 and (now - newest) > days * 86400 

163 

164 

165def main() -> None: 

166 """Entry for the detached child: argv is (cache_dir, skip_stem).""" 

167 if len(sys.argv) < 2: 

168 return 

169 skip = sys.argv[2] if len(sys.argv) > 2 else "" 

170 collect(Path(sys.argv[1]), skip)