From b842a4990ac7f356ccd7d1276f0c64eee85b05b7 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:56:27 +0000 Subject: [PATCH 1/8] fork-kit: pc-agents walls and their check (part 1 of 2) A WSL2 distro on a Windows PC as the wall around T3's agents: the files the kit owns (wsl.conf, resolv.conf, managed policy, uid-matched nft table, a root system unit with User=agent and a sandbox, a daily check timer), rendered from the pc-agents section of the machine file, and `pc-agents check [--enforce]`: the fleet's wsl2 walls (run, not copied) plus the kit's own. The unit runs the check before every start. A server test pins that the managed policy locks the fork's settings (agentSlots, linkedEnvironmentsEnabled, automaticUpdates). Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 1 + .../policy/managedSettings.forkKeys.test.ts | 85 +++++++ fork-kit/pc-agents/README.md | 94 ++++++++ fork-kit/pc-agents/lib/pc_agents.py | 219 ++++++++++++++++++ fork-kit/pc-agents/sbin/pc-agents | 115 +++++++++ fork-kit/pc-agents/tests/check.test.py | 76 ++++++ fork-kit/pc-agents/tests/fake_distro.py | 157 +++++++++++++ 7 files changed, 747 insertions(+) create mode 100644 apps/server/src/policy/managedSettings.forkKeys.test.ts create mode 100644 fork-kit/pc-agents/README.md create mode 100644 fork-kit/pc-agents/lib/pc_agents.py create mode 100755 fork-kit/pc-agents/sbin/pc-agents create mode 100644 fork-kit/pc-agents/tests/check.test.py create mode 100644 fork-kit/pc-agents/tests/fake_distro.py diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 58e58018f94aa..0e2b9a7491353 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -56,6 +56,7 @@ fork-kit/heavy/tests/install.test.py fork-kit/heavy/tests/sweep.test.py # A real systemd user manager (the step starts one for the runner user). fork-kit/heavy/tests/heavy.test.py run by .github/workflows/fork-tests.yml +fork-kit/pc-agents/tests/check.test.py # Upstream's tests, run where fork-tests.yml follows upstream's ci.yml. .github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml diff --git a/apps/server/src/policy/managedSettings.forkKeys.test.ts b/apps/server/src/policy/managedSettings.forkKeys.test.ts new file mode 100644 index 0000000000000..99f818179d0ab --- /dev/null +++ b/apps/server/src/policy/managedSettings.forkKeys.test.ts @@ -0,0 +1,85 @@ +// Fork: a managed policy locks the fork's own settings like upstream's. The pc-agents host kit +// (fork-kit/pc-agents) writes a policy that locks the beta switch, automatic updates and the +// child ceiling (agentSlots, one field per dotted path); this pins that each of them is accepted +// and held, and that a partial object is refused (the kit refuses object values for that reason). + +import { DEFAULT_SERVER_SETTINGS, type ServerSettings } from "@t3tools/contracts"; +import { assert, it } from "@effect/vitest"; + +import { makeManagedSettings } from "./managedSettings.ts"; + +const KIT_POLICY = { + linkedEnvironmentsEnabled: { value: true, locked: true }, + automaticUpdates: { value: false, locked: true }, + "agentSlots.enabled": { value: true, locked: true }, + "agentSlots.maxPerMachine": { value: 2, locked: true }, +}; + +const managed = () => { + const result = makeManagedSettings(KIT_POLICY, "full-access"); + if (result._tag !== "Valid") assert.fail(result.reason); + return result.settings; +}; + +const userSettings: ServerSettings = { + ...DEFAULT_SERVER_SETTINGS, + linkedEnvironmentsEnabled: false, + automaticUpdates: true, + agentSlots: { ...DEFAULT_SERVER_SETTINGS.agentSlots, enabled: false, maxPerMachine: 9 }, +}; + +it("accepts the pc-agents kit's locks on fork settings", () => { + assert.deepEqual([...managed().lockedPaths].toSorted(), [ + "agentSlots.enabled", + "agentSlots.maxPerMachine", + "automaticUpdates", + "linkedEnvironmentsEnabled", + ]); +}); + +it("lays the locked fork values over the user's", () => { + const applied = managed().applyLocks(userSettings); + assert.strictEqual(applied.linkedEnvironmentsEnabled, true); + assert.strictEqual(applied.automaticUpdates, false); + assert.strictEqual(applied.agentSlots.enabled, true); + assert.strictEqual(applied.agentSlots.maxPerMachine, 2); +}); + +it("refuses a write that raises the locked child ceiling or turns the beta off", () => { + const settings = managed(); + const current = settings.applyLocks(userSettings); + const raise = settings.vetWrite( + current, + { ...current, agentSlots: { ...current.agentSlots, maxPerMachine: 5 } }, + { agentSlots: { maxPerMachine: 5 } }, + ); + assert.deepEqual(raise, { _tag: "Locked", path: "agentSlots.maxPerMachine" }); + const off = settings.vetWrite( + current, + { ...current, linkedEnvironmentsEnabled: false }, + { linkedEnvironmentsEnabled: false }, + ); + assert.deepEqual(off, { _tag: "Locked", path: "linkedEnvironmentsEnabled" }); +}); + +it("locks one field of a fork setting and leaves the others to the user", () => { + const result = makeManagedSettings({ "agentSlots.maxPerMachine": { value: 2, locked: true } }); + if (result._tag !== "Valid") assert.fail(result.reason); + assert.strictEqual(result.settings.applyLocks(userSettings).agentSlots.maxPerMachine, 2); + assert.strictEqual(result.settings.applyLocks(userSettings).agentSlots.enabled, false); +}); + +it("takes an object only as a fork setting's complete value", () => { + const partial = makeManagedSettings({ + agentSlots: { value: { maxPerMachine: 2 }, locked: true }, + }); + assert.strictEqual(partial._tag, "Invalid"); + const whole = makeManagedSettings({ + agentSlots: { + value: { ...DEFAULT_SERVER_SETTINGS.agentSlots, maxPerMachine: 2 }, + locked: true, + }, + }); + if (whole._tag !== "Valid") assert.fail(whole.reason); + assert.strictEqual(whole.settings.applyLocks(userSettings).agentSlots.maxPerMachine, 2); +}); diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md new file mode 100644 index 0000000000000..b825f0888c34a --- /dev/null +++ b/fork-kit/pc-agents/README.md @@ -0,0 +1,94 @@ +# pc-agents: run T3's agents on a Windows PC, walled in a WSL2 distro of their own + +A PC can take agent work from another T3 (a linked environment, with **Full access**). Full access runs as the PC +T3's user, so the link is not the wall: the distro is. This kit makes a fresh WSL2 distro into that wall and runs +T3 in it as `agent`, a user with no sudo, under a root-owned system unit. Everything that holds the wall is root's, +so the agent cannot undo it: + +| Wall | What the kit does | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` | +| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | +| No way out of the user | one login user (`agent`), in no group but its own; cron, at and ssh masked | +| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | +| No LAN, no Windows host | nftables, for the agent's uid only: private, link-local, CGNAT (Tailscale) and multicast ranges dropped, so the Windows host (the NAT gateway), the home network and the tailnet are out of reach. Public addresses (GitHub, Anthropic, package mirrors) and replies to the linked T3 go through | +| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked | +| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and daily (a failed wall stops T3) | + +`pc-agents check` is the fleet's WSL2 walls check (`.github/scripts/fork-fleet-walls.py` with +`fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: every file above is root's, +mode 644 and exactly what the kit writes, nothing overrides the unit (no drop-in), the masks are in place, `agent` is +the only login user, the kit's code is root's, no Windows drive is mounted, the firewall table is loaded, and the +default gateway (the Windows host) is in a blocked range. The fleet's `mounts`, `sockets` and `userns` walls are +about the CI launcher's job units, so they are not reported here; the unit's own sandbox is checked instead (it must +be the kit's file, byte for byte, with no drop-in). + +Linux (WSL2, systemd, Debian or Ubuntu) only. Python 3 standard library only. This folder has the walls and their +check so far; `pc-agents setup`, which writes them into a fresh distro, and its owner steps are still to come. + +## Config + +The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine.json` (root-owned, see +[common](../common/README.md)). Example (made-up values): + +```json +{ + "version": 1, + "pc-agents": { + "organization": "Example home lab", + "t3Version": "0.0.30", + "t3ReleaseBaseUrl": "https://github.com/example/t3code/releases/download", + "port": 3773, + "maxRuntimeMode": "full-access", + "lockedSettings": { "agentSlots.enabled": true, "agentSlots.maxPerMachine": 2 }, + "memoryMax": "8G", + "cpuQuota": "400%", + "nameservers": ["1.1.1.1", "9.9.9.9"], + "tailscaleTag": "tag:pc-agents" + } +} +``` + +| Key | What | +| ----------------------- | ----------------------------------------------------------------------------------------------------------- | +| `organization` | the name the managed policy shows in T3's settings | +| `t3Version` | the exact T3 version to run (the same as the T3 that links to this PC); installed with `scripts/install.sh` | +| `t3ReleaseBaseUrl` | where that version's release files are (`T3CODE_RELEASE_BASE_URL` of `scripts/install.sh`) | +| `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | +| `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | +| `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | +| `lockedSettings` | more T3 settings to lock, `{path: value}`, one field per dotted path: the child ceiling (`agentSlots.…`) | +| `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | +| `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | +| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default | +| `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | + +Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`, +`usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's +(`apps/server/src/policy/managedSettings.ts`), and a test proves it for the fork's keys. Lock one field per dotted +path (`agentSlots.maxPerMachine`): T3 takes an object only as a setting's complete value and refuses to start on a +policy it cannot apply, so the kit refuses object values. A field inside a setting that can be empty must be locked +as the whole setting; T3 says so in its log (`journalctl -u t3code-agents`) and does not start. + +## Limits + +- **One VM for all distros.** WSL2 runs every distro in one VM: one kernel, one network namespace, one memory cap. + The firewall table is therefore global and matches the agent's uid in every distro (keep `agentUid` apart from + the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable. + Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. +- **Public internet is open** to the agent: the firewall blocks private ranges, not the public internet (GitHub's + and Anthropic's addresses are many and change). Add an outbound proxy if that matters. +- **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not + run; the distro is the sandbox, so use full access. +- **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, + but the code files are not hashed. + +## Tests + +``` +python3 -I -B fork-kit/pc-agents/tests/check.test.py +``` + +A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake `nft` and +`systemctl`: every wall passes on a walled distro, and each one fails on its own when its part is taken away. +`apps/server/src/policy/managedSettings.forkKeys.test.ts` proves the managed policy locks the fork's settings. diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py new file mode 100644 index 0000000000000..29000cb399cb1 --- /dev/null +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -0,0 +1,219 @@ +"""pc-agents: the facts, the files this kit owns and the checks, shared by `pc-agents setup` and `pc-agents check`. + +Every path is read and written under ROOT, which is '' (the real machine) except in the tests, where +PC_AGENTS_ROOT names a fake one; root never gets a fake one. Standard library only.""" +import hashlib, ipaddress, json, os, pwd, re, subprocess + +import machine_file + +KIT = 'pc-agents' +ROOT = os.environ.get('PC_AGENTS_ROOT', '') +if ROOT and os.geteuid() == 0: + raise SystemExit('pc-agents: PC_AGENTS_ROOT is for the tests only, never for root; unset it') +OWNER = os.geteuid() if ROOT else 0 # who must own every file the kit writes +HOME, USER = '/home/agent', 'agent' +OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' +RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote +UNIT, WALLS = 't3code-agents.service', 'pc-agents-walls' +CHECK = OPT + '/current/sbin/pc-agents' +WALLS_PY = OPT + '/current/walls/fork-fleet-walls.py' +# Walls of fork-fleet-walls/wsl2 that are about the fleet launcher's job units. Here the job unit is T3's own unit, +# whose sandbox this kit writes and checks itself (the unit file must be exactly the kit's, with no drop-in). +UNIT_WALLS = {'mounts', 'sockets', 'userns'} +MASKS = ['/etc/binfmt.d/WSLInterop.conf'] + ['/etc/systemd/system/' + u for u in + ('cron.service', 'atd.service', 'ssh.service', 'ssh.socket')] +UNIT_DIRS = ('/etc/systemd/system', '/run/systemd/system', '/usr/local/lib/systemd/system', '/usr/lib/systemd/system', + '/lib/systemd/system', '/run/systemd/transient', '/run/systemd/generator.early') +BLOCKED4 = ('0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '169.254.0.0/16', '172.16.0.0/12', '192.168.0.0/16', + '224.0.0.0/4', '240.0.0.0/4') +BLOCKED6 = ('fc00::/7', 'fe80::/10', 'ff00::/8') +MODES = ('approval-required', 'auto-accept-edits', 'auto', 'full-access') +KIT_SETTINGS = {'linkedEnvironmentsEnabled': True, 'automaticUpdates': False} +VERSION = re.compile(r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?') +HEAD = '# pc-agents kit: written by `pc-agents setup`; an edit is refused at its next run and fails the walls check\n' + +# What the kit runs from a release: source path -> (path in the version folder, mode). The walls are the merged +# fleet walls of the same release, not a copy: fork-fleet-walls.py reads its folder and the launcher beside it. +CODE = {'fork-kit/pc-agents/sbin/pc-agents': ('sbin/pc-agents', 0o755), + 'fork-kit/pc-agents/lib/pc_agents.py': ('lib/pc_agents.py', 0o644), + 'fork-kit/common/machine_file.py': ('lib/machine_file.py', 0o644), + '.github/scripts/fork-fleet-walls.py': ('walls/fork-fleet-walls.py', 0o755), + '.github/scripts/fork-fleet-launch.sh': ('walls/fork-fleet-launch.sh', 0o755), + '.github/scripts/fork-fleet-job-paths.txt': ('walls/fork-fleet-job-paths.txt', 0o644), + 'scripts/install.sh': ('share/install.sh', 0o644)} +for _w, _m in (('linux', 0o755), ('wsl2', 0o755), ('linux.fixture.json', 0o644), ('wsl2.fixture.json', 0o644)): + CODE['.github/scripts/fork-fleet-walls/' + _w] = ('walls/fork-fleet-walls/' + _w, _m) + + +def is_ip_list(value, net, glob=False, most=64): + try: + ips = [(ipaddress.ip_network if net else ipaddress.ip_address)(v) for v in value] + except (TypeError, ValueError) as e: + return f'not a list of IP {"networks" if net else "addresses"} ({e})' + if not 0 < len(ips) <= most and not (net and not ips): + return f'1 to {most} entries' + return 'only public addresses (a private one is blocked)' if glob and not all(i.is_global for i in ips) else None + + +SPEC = { + 'organization': (str, machine_file.REQUIRED, lambda v: None if 0 < len(v) <= 80 else '1 to 80 characters'), + 't3Version': (str, machine_file.REQUIRED, lambda v: None if VERSION.fullmatch(v) else 'an exact version, no v'), + 't3ReleaseBaseUrl': (str, machine_file.REQUIRED, + lambda v: None if re.fullmatch(r'https://[A-Za-z0-9./_-]+', v) else 'an https URL'), + 'port': (int, machine_file.REQUIRED, lambda v: None if 1024 <= v <= 65535 else '1024 to 65535'), + 'agentUid': (int, 1100, lambda v: None if 1000 <= v < 60000 else '1000 to 59999'), + 'maxRuntimeMode': (str, machine_file.REQUIRED, lambda v: None if v in MODES else 'one of ' + ', '.join(MODES)), + 'lockedSettings': (dict, {}, lambda v: next((f'{k}: set by the kit itself, or not a settings path' for k in v + if k in KIT_SETTINGS or not re.fullmatch(r'[a-z]\w*(\.\w+)*', k)), None) + or next((f'{k}: lock one field per dotted path ({k}.field), not an object (T3 takes an object ' + 'only as the whole setting and does not start on a policy it cannot apply)' + for k, x in v.items() if isinstance(x, dict)), None)), + 'memoryMax': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*[KMGT]', v) else 'like 8G'), + 'cpuQuota': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*%', v) else 'like 400%'), + 'nameservers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, False, True, 3)), + 'allow': (list, [], lambda v: is_ip_list(v, True)), + 'tailscaleTag': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'tag:[a-z0-9-]{1,50}', v) else 'tag:name'), +} + + +def config(): + section = machine_file.read_section(KIT, ROOT + machine_file.default_path() if ROOT else None, OWNER) + return machine_file.fields(section, SPEC, f'{machine_file.default_path()}: {KIT}') + + +def runtime(c): + return f'{HOME}/.t3/runtime/versions/{c["t3Version"]}/t3' + + +def render(c): + """{path: (text, mode)} of every file the kit owns, from the config alone (the same every run).""" + settings = {**KIT_SETTINGS, **c['lockedSettings']} + policy = {'schemaVersion': 1, 'organization': c['organization'], + 'settings': {k: {'value': v, 'locked': True} for k, v in settings.items()}, + 'limits': {'maxRuntimeMode': c['maxRuntimeMode'], 'connect': False, 'webhookTasks': False, + 'browserCookieImport': False, 'updates': {'mode': 'off'}}} + allow4 = [a for a in c['allow'] if ':' not in a] + allow6 = [a for a in c['allow'] if ':' in a] + nft = HEAD + 'table inet pc_agents\ndelete table inet pc_agents\ntable inet pc_agents {\n\tchain agent_out {\n' + \ + '\t\ttype filter hook output priority filter; policy accept;\n' + \ + f'\t\tmeta skuid != {c["agentUid"]} accept\n\t\tct state established,related accept\n\t\toifname "lo" accept\n' + \ + (f'\t\tip daddr {{ {", ".join(allow4)} }} accept\n' if allow4 else '') + \ + (f'\t\tip6 daddr {{ {", ".join(allow6)} }} accept\n' if allow6 else '') + \ + f'\t\tip daddr {{ {", ".join(BLOCKED4)} }} drop\n\t\tip6 daddr {{ {", ".join(BLOCKED6)} }} drop\n\t}}\n}}\n' + unit = HEAD + f'''[Unit] +Description=T3 Code for agents, walled (pc-agents kit) +Wants=network-online.target +After=network-online.target + +[Service] +User={USER} +Group={USER} +Environment=HOME={HOME} T3CODE_HOME={HOME}/.t3 T3CODE_PORT={c["port"]} T3CODE_LAN=true +Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin +ExecStartPre=+/usr/sbin/nft -f {STATE}/firewall.nft +ExecStartPre=+{CHECK} check +ExecStart={runtime(c)} serve +Restart=on-failure +RestartSec=30 +UMask=0077 +MemoryMax={c["memoryMax"]} +CPUQuota={c["cpuQuota"]} +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths={HOME} +PrivateTmp=yes +PrivateDevices=yes +PrivateIPC=yes +ProtectProc=invisible +InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK +SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS +SystemCallArchitectures=native +RestrictNamespaces=yes +RestrictSUIDSGID=yes +RestrictRealtime=yes +LockPersonality=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes + +[Install] +WantedBy=multi-user.target +''' + wsl = HEAD + ('[boot]\nsystemd=true\n[automount]\nenabled=false\nmountFsTab=false\n[interop]\nenabled=false\n' + f'appendWindowsPath=false\n[network]\ngenerateResolvConf=false\n[user]\ndefault={USER}\n') + sd = '/etc/systemd/system/' + return { + '/etc/wsl.conf': (wsl, 0o644), + '/etc/resolv.conf': (HEAD + ''.join(f'nameserver {n}\n' for n in c['nameservers']), 0o644), + '/etc/t3code/policy.json': (json.dumps(policy, indent=2, sort_keys=True) + '\n', 0o644), + STATE + '/firewall.nft': (nft, 0o644), + sd + UNIT: (unit, 0o644), + sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents daily walls check\n\n[Service]\nType=oneshot\n' + f'ExecStart={CHECK} check --enforce\n', 0o644), + sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents daily walls check\n\n[Timer]\nOnCalendar=daily\n' + 'Persistent=true\n\n[Install]\nWantedBy=timers.target\n', 0o644), + } + + +def sha(text): + return hashlib.sha256(text.encode() if isinstance(text, str) else text).hexdigest() + + +def record(): + try: + with open(ROOT + RECORD, encoding='utf-8') as f: + return dict(reversed(line.rstrip('\n').split('\t', 1)) for line in f if '\t' in line) + except FileNotFoundError: + return {} + + +def lstat(path): + try: + return os.lstat(ROOT + path) + except FileNotFoundError: + return None + + +def safe(path, st, kind): + """Why path (stat st) is not what kind says (S_ISREG etc.), owned by OWNER, writable by it alone; or None.""" + if st is None: + return f'{path} is missing' + if not kind(st.st_mode) or st.st_uid != OWNER or st.st_mode & 0o022: + return f'{path} is not owned by uid {OWNER} and writable by it alone (or is the wrong kind of file)' + return None + + +def users(): + """{name: (uid, gid)} of the passwd file, and {group: [members]}.""" + def rows(path): + try: + with open(ROOT + path, encoding='utf-8') as f: + return [line.rstrip('\n').split(':') for line in f if line.count(':') >= 3] + except FileNotFoundError: + return [] + return ({r[0]: (int(r[2]), int(r[3])) for r in rows('/etc/passwd') if r[2].isdigit() and r[3].isdigit()}, + {r[0]: [m for m in r[3].split(',') if m] for r in rows('/etc/group')}) + + +def user_problem(c, need_agent): + """Why this distro is someone else's (another login user, an agent that is not the kit's), or None.""" + people, groups = users() + others = sorted(n for n, (uid, _) in people.items() if 1000 <= uid < 60000 and n != USER) + if others: + return f'this distro already has login users ({", ".join(others)}): use a fresh distro for pc-agents' + if USER not in people: + return f'no user {USER}' if need_agent else None + if people[USER][0] != c['agentUid']: + return f'user {USER} has uid {people[USER][0]}, not {c["agentUid"]}: not the kit\'s' + extra = sorted(g for g, members in groups.items() if USER in members) + return f'user {USER} is in other groups ({", ".join(extra)}): it must have none' if extra else None + + +def run(argv, **kw): + return subprocess.run([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=120, **kw) diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents new file mode 100755 index 0000000000000..449635b942ef4 --- /dev/null +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -0,0 +1,115 @@ +#!/usr/bin/python3 -I +"""pc-agents: wall a WSL2 distro in, then run T3 Code in it as an unprivileged user (fork-kit/pc-agents/README.md). + + pc-agents check [--enforce] every wall, PASS or FAIL; exit 0 only when all hold. --enforce also + stops T3 when one fails (the daily timer). T3's unit runs it before + every start, so T3 never starts on a failed wall. +Standard library only.""" +import os, re, stat, sys + +HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +COMMON = os.path.join(os.path.dirname(HERE), 'common') +sys.path[:0] = [os.path.join(HERE, 'lib')] + ([COMMON] if os.path.isdir(COMMON) else []) +sys.dont_write_bytecode = True +import pc_agents as k # noqa: E402 + + +def say(text): + print(f'pc-agents: {text}', flush=True) + + +def own_walls(c): + """(wall, what, why or None) for the walls this kit adds to fork-fleet-walls/wsl2.""" + out, done = [], k.record() + for path, (text, mode) in k.render(c).items(): + st = k.lstat(path) + why = k.safe(path, st, stat.S_ISREG) + if why is None: + with open(k.ROOT + path, 'rb') as f: + why = None if f.read() == text.encode() and done.get(path) == k.sha(text) else \ + f'{path} differs from what the kit writes (pc-agents setup)' + out.append(('file:' + os.path.basename(path), f'{path} is the kit\'s, root-owned', why)) + masks = [p for p in k.MASKS if not (k.lstat(p) and os.path.islink(k.ROOT + p) + and os.readlink(k.ROOT + p) == '/dev/null')] + out.append(('masks', 'WSL interop, cron, at and ssh are masked', masks and 'not masked: ' + ', '.join(masks))) + extra = [f'{d}/{n}' for d in k.UNIT_DIRS for n in (k.UNIT + '.d', k.WALLS + '.service.d') + ( + ('service.d',) if d in k.UNIT_DIRS[:2] else ()) + ((k.UNIT,) if d != k.UNIT_DIRS[0] else ()) + if k.lstat(f'{d}/{n}')] + out.append(('drop-ins', 'nothing overrides the unit', extra and 'found ' + ', '.join(extra))) + out.append(('agent', f'one login user, {k.USER}, in no group but its own', k.user_problem(c, True))) + code = [] + for top, dirs, names in os.walk(k.ROOT + k.OPT + '/current/'): + for n in dirs + names: + p = os.path.join(top, n)[len(k.ROOT):] + code += [why] if (why := k.safe(p, k.lstat(p), lambda m: stat.S_ISDIR(m) or stat.S_ISREG(m))) else [] + current = k.lstat(k.OPT + '/current') + code += [] if current and current.st_uid == k.OWNER else [f'{k.OPT}/current is not the kit\'s'] + out.append(('kit-code', f'{k.OPT}/current is root-owned', code[:3] and '; '.join(code[:3]))) + mounts = [] + try: + with open(k.ROOT + '/proc/self/mountinfo', encoding='utf-8', errors='replace') as f: + for line in f: + fields = line.split(' - ') + where, fstype = fields[0].split()[4], (fields[1].split() or [''])[0] if len(fields) > 1 else '' + if fstype in ('9p', 'drvfs', 'virtiofs') and not where.startswith('/usr/lib/wsl'): + mounts.append(f'{where} ({fstype})') + except (OSError, IndexError) as e: + mounts.append(f'cannot read /proc/self/mountinfo ({e})') + out.append(('automount', 'no Windows drive is mounted', mounts and 'mounted: ' + ', '.join(mounts[:3]))) + r = k.run(['/usr/sbin/nft', 'list', 'table', 'inet', 'pc_agents']) + nft_ok = r.returncode == 0 and f'skuid != {c["agentUid"]}' in r.stdout and ' drop' in r.stdout + out.append(('firewall', 'the agent firewall is loaded', None if nft_ok else 'table inet pc_agents is not loaded')) + gateway = None + try: + with open(k.ROOT + '/proc/net/route', encoding='utf-8') as f: + row = next((line.split() for line in f if line.split()[1:2] == ['00000000']), None) + gateway = row and k.ipaddress.ip_address(bytes.fromhex(row[2])[::-1]) + except (OSError, ValueError) as e: + gateway = f'unknown ({e})' + blocked = isinstance(gateway, k.ipaddress.IPv4Address) and \ + any(gateway in k.ipaddress.ip_network(n) for n in k.BLOCKED4) + out.append(('gateway', 'the Windows host (the default gateway) is in a blocked range', + None if blocked else f'the gateway is {gateway}, not in a blocked range')) + return out + + +def wsl2_walls(): + """(wall, what, why or None) of the fleet's wsl2 walls (fork-fleet-walls.py, its contract enforced there).""" + env = {'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'LC_ALL': 'C'} + if k.ROOT: + env['FORK_FLEET_WALLS_ROOT'] = k.ROOT + try: + p = k.subprocess.run([k.ROOT + k.WALLS_PY, 'describe'], env=env, stdin=k.subprocess.DEVNULL, + capture_output=True, text=True, timeout=120, cwd='/') + except (OSError, k.subprocess.TimeoutExpired) as e: + return [('wsl2', 'the fleet\'s wsl2 walls ran', f'cannot run them ({e})')] + lines = [m.groups() for m in re.finditer(r'^(PASS|FAIL) (\S+) (.*)$', p.stdout, re.M)] + if p.returncode not in (0, 1) or not lines: + return [('wsl2', 'the fleet\'s wsl2 walls ran', f'exit {p.returncode}: {p.stderr.strip()[-300:]}')] + return [(f'wsl2:{w}', what, None if mark == 'PASS' else what) for mark, w, what in lines if w not in k.UNIT_WALLS] + + +def check(enforce): + try: + c = k.config() + results = wsl2_walls() + own_walls(c) + except Exception as e: # a wall that cannot be told does not hold + results = [('check', 'the walls can be read', f'{e.__class__.__name__}: {e}')] + for wall, what, why in results: + print(f'FAIL {wall} {what}: {why}'[:500] if why else f'PASS {wall} {what}', flush=True) + failed = any(why for _, _, why in results) + if failed and enforce: + k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) + say(f'a wall does not hold: stopped {k.UNIT}') + return 1 if failed else 0 + + +def main(argv): + if argv[:1] == ['check'] and argv[1:] in ([], ['--enforce']): + return check(argv[1:] == ['--enforce']) + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py new file mode 100644 index 0000000000000..03fc4bb708724 --- /dev/null +++ b/fork-kit/pc-agents/tests/check.test.py @@ -0,0 +1,76 @@ +"""pc-agents check on a fake WSL2 distro (fake_distro.py): every wall passes on a walled distro, and each one fails +on its own when its part of the wall is taken away; --enforce stops T3. + + python3 -I -B fork-kit/pc-agents/tests/check.test.py +""" +import os, shutil, sys, unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from fake_distro import Case, DRVFS, SHA1, k # noqa: E402 + + +class Walls(Case): + def setUp(self): + super().setUp() + self.walled() + + def fails(self, wall, *also): + """check fails on wall, and on no other wall but those in also (which the same change takes away).""" + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertEqual({line.split()[1] for line in out.splitlines() if line.startswith('FAIL')}, {wall, *also}, out) + + def test_all_hold(self): + code, out = self.check() + self.assertEqual(code, 0, out) + walls = [line.split()[1] for line in out.splitlines()] + for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', + 'drop-ins', 'agent', 'kit-code', 'automount', 'firewall', 'gateway', 'file:policy.json', + 'file:t3code-agents.service', 'file:firewall.nft', 'file:wsl.conf'): + self.assertIn(wall, walls) + for wall in k.UNIT_WALLS: # the launcher's job units: not T3's unit + self.assertNotIn('wsl2:' + wall, walls) + + def test_enforce_stops_t3(self): + os.unlink(self.path('/log/nft-loaded')) + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('stop t3code-agents.service', self.log()) + + def test_each_wall(self): + def edit(p, old, new): + return lambda: self.write(p, self.read(p).replace(old, new)) + cases = { + 'file:policy.json': edit('/etc/t3code/policy.json', '"full-access"', '"auto"'), + 'file:t3code-agents.service': lambda: os.chmod(self.path('/etc/systemd/system/t3code-agents.service'), + 0o666), + 'file:resolv.conf': lambda: os.unlink(self.path('/etc/resolv.conf')), + 'masks': lambda: os.unlink(self.path('/etc/systemd/system/ssh.socket')), + 'drop-ins': lambda: os.makedirs(self.path('/etc/systemd/system/t3code-agents.service.d')), + 'agent': edit('/etc/group', 'sudo:x:27:', 'sudo:x:27:agent'), + 'kit-code': lambda: os.chmod(self.path(f'/opt/pc-agents/{SHA1}/lib/pc_agents.py'), 0o666), + 'automount': lambda: self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo') + DRVFS), + 'firewall': lambda: os.unlink(self.path('/log/nft-loaded')), + 'gateway': lambda: self.write('/proc/net/route', 'Iface\tDestination\tGateway\nx\t00000000\t08080808\n'), + 'wsl2:interop': lambda: self.write('/proc/sys/fs/binfmt_misc/WSLInterop', + 'enabled\ninterpreter /init\nflags: PF\noffset 0\nmagic 4d5a\n'), + 'wsl2:networking': edit('/usr/bin/wslinfo', 'echo nat', 'echo mirrored'), + } + clean = {p: v[4] for p, v in self.snapshot().items()} + for wall, change in cases.items(): + with self.subTest(wall=wall): + shutil.rmtree(self.root) + self.setUp() + self.assertEqual({p: v[4] for p, v in self.snapshot().items()}, clean) + change() + self.fails(wall, *(['wsl2:resolv'] if wall == 'file:resolv.conf' else [])) + + def test_no_config_fails(self): + os.unlink(self.path('/etc/fork-kit/machine.json')) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL check ', out) + + +if __name__ == '__main__': + unittest.main() diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py new file mode 100644 index 0000000000000..abe99cb8ce8ea --- /dev/null +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -0,0 +1,157 @@ +"""A fake WSL2 distro in a temporary folder for the pc-agents tests: every fixed path under PC_AGENTS_ROOT, the +current user as root, fake useradd, runuser, systemctl and nft (each logs to /log in the fake root). No root, no +systemd, no network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" +import contextlib, importlib.machinery, importlib.util, io, json, os, shutil, stat, sys, tempfile, unittest + +KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +REPO = os.path.dirname(os.path.dirname(KIT)) +WSL2 = os.path.join(REPO, '.github', 'scripts', 'fork-fleet-walls', 'wsl2.fixture.json') +SHA1, SHA2 = 'a' * 40, 'b' * 40 +CONFIG = {'organization': 'Example lab', 't3Version': '1.2.3', 't3ReleaseBaseUrl': 'https://example.com/releases', + 'port': 3800, 'maxRuntimeMode': 'full-access', 'lockedSettings': {'agentSlots.maxPerMachine': 2}, + 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'tailscaleTag': 'tag:pc-agents'} +TOOLS = { # fake programs, under the fake root; each finds the root from its own path + 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; prev=; for a; do [ "$prev" = --uid ] && uid=$a; prev=$a; ' + 'done\necho "$a:x:$uid:$uid::/home/$a:/bin/bash" >>"$R/etc/passwd"\n' + 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', + 'usr/sbin/runuser': 'R="${0%/usr/sbin/runuser}"; echo "$*" >>"$R/log/runuser"\n[ -e "$R/log/runuser-fail" ] && exit 1\n' + 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' + 'd="$R/home/agent/.t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n" >"$d/t3"\n', + 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"\n', + 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' + 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', +} +DRVFS = '43 29 0:43 / /mnt/c rw,noatime - 9p drvfs rw\n44 29 0:44 / /c rw,noatime - 9p drvfs rw\n' + + +def blob(path): + with open(path, 'rb') as f: + return f.read() + + +def load(): + os.environ['PC_AGENTS_ROOT'] = '/nonexistent' + loader = importlib.machinery.SourceFileLoader('pc_agents_cli', os.path.join(KIT, 'sbin', 'pc-agents')) + mod = importlib.util.module_from_spec(importlib.util.spec_from_loader('pc_agents_cli', loader)) + loader.exec_module(mod) + return mod + + +cli = load() +k = cli.k + + +class Case(unittest.TestCase): + def setUp(self): + self.umask = os.umask(0o022) + self.root = os.path.realpath(tempfile.mkdtemp(prefix='pc-agents-test-')) + os.chmod(self.root, 0o755) + self.addCleanup(self.close) + k.ROOT, k.OWNER = self.root, os.geteuid() + with open(WSL2, encoding='utf-8') as f: + files = json.load(f)['pass'] + for rel in ('etc/wsl.conf', 'etc/resolv.conf', 'etc/binfmt.d/WSLInterop.conf'): + del files[rel] # a fresh distro: the kit writes these + files['etc/wsl.conf'] = '[boot]\nsystemd=true\n' + files['etc/passwd'] = 'root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534::/nonexistent:/usr/sbin/nologin\n' + files['etc/group'] = 'root:x:0:\nsudo:x:27:\nnogroup:x:65534:\n' + files['proc/net/route'] = 'Iface\tDestination\tGateway\tFlags\nboth\t00000000\t011014AC\t0003\n' + files['log/'] = '' + files.update({rel: {'text': '#!/bin/sh\n' + text, 'mode': '755'} for rel, text in TOOLS.items()}) + for rel, value in files.items(): + self.write('/' + rel, value) + os.symlink('/mnt/wsl/resolv.conf', self.path('/etc/resolv.conf')) + self.machine(CONFIG) + + def close(self): + os.umask(self.umask) + shutil.rmtree(self.root, ignore_errors=True) + + def path(self, p): + return self.root + p + + def write(self, p, value): + full = self.path(p) + if os.path.islink(full): + os.unlink(full) + if p.endswith('/'): + return os.makedirs(full, exist_ok=True) + os.makedirs(os.path.dirname(full), exist_ok=True) + text, mode = (value['text'], int(value['mode'], 8)) if isinstance(value, dict) else (value, 0o644) + with open(full, 'w', encoding='utf-8') as f: + f.write(text) + os.chmod(full, mode) + + def read(self, p): + with open(self.path(p), encoding='utf-8') as f: + return f.read() + + def machine(self, section): + self.write('/etc/fork-kit/machine.json', json.dumps({'version': 1, 'pc-agents': section})) + + def terminate(self): + """What `wsl --terminate` does once wsl.conf turned automount off: Windows' drives are no longer mounted.""" + self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo').replace(DRVFS, '')) + + def call(self, fn, *argv): + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + code = fn(*argv) + return code, out.getvalue() + + def setup_run(self, sha=SHA1): + return self.call(cli.main, ['setup', '--source', REPO, '--sha', sha]) + + def check(self, *flags): + return self.call(cli.main, ['check', *flags]) + + def log(self, name='systemctl'): + try: + return self.read('/log/' + name).splitlines() + except FileNotFoundError: + return [] + + def ready(self): + """A distro set up all the way: first run, `wsl --terminate`, second run.""" + code, out = self.setup_run() + self.assertEqual(code, 3, out) + self.terminate() + code, out = self.setup_run() + self.assertEqual(code, 0, out) + + def walled(self, sha=SHA1): + """What `pc-agents setup` leaves after `wsl --terminate` and a second run, written here directly.""" + files = k.render(k.config()) + for p, (text, mode) in files.items(): + self.write(p, {'text': text, 'mode': f'{mode:o}'}) + self.write(k.RECORD, ''.join(f'{k.sha(t)}\t{p}\n' for p, (t, _) in sorted(files.items()))) + for p in k.MASKS: + os.makedirs(os.path.dirname(self.path(p)), exist_ok=True) + os.symlink('/dev/null', self.path(p)) + with open(self.path('/etc/passwd'), 'a') as f: + f.write('agent:x:1100:1100::/home/agent:/bin/bash\n') + with open(self.path('/etc/group'), 'a') as f: + f.write('agent:x:1100:\n') + for src, (dst, mode) in k.CODE.items(): + with open(os.path.join(REPO, src), encoding='utf-8') as f: + self.write(f'/opt/pc-agents/{sha}/{dst}', {'text': f.read(), 'mode': f'{mode:o}'}) + os.symlink(sha, self.path('/opt/pc-agents/current')) + shutil.copy(self.path(k.STATE + '/firewall.nft'), self.path('/log/nft-loaded')) + self.terminate() + + def snapshot(self): + """Every path under the fake root but the fake tools' logs: kind, mode, owner, mtime, inode, link, content.""" + out = {} + for top, dirs, names in os.walk(self.root): + for n in dirs + names: + full = os.path.join(top, n) + rel = full[len(self.root):] + if rel.startswith('/log'): + continue + st = os.lstat(full) + body = os.readlink(full) if stat.S_ISLNK(st.st_mode) else \ + blob(full) if stat.S_ISREG(st.st_mode) else None + out[rel] = (st.st_mode, st.st_uid, st.st_mtime_ns, st.st_ino, body) + return out + + From 364edb63fe5f325607d807f7a23c3b2aab2ab625 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:10:16 +0000 Subject: [PATCH 2/8] fork-kit: pc-agents fix round 1 (walls) - Agent code runs only in T3's unit: default distro user root, the agent's user manager masked, walls for the mask, linger and user units. - T3's runtime is root's under /opt/pc-agents/t3; the unit runs it from there; a wall checks its owner, mode and recorded hash. - A check that cannot run or crashes stops T3 (ExecStopPost on the walls service); a failed stop is reported. - Second network wall in the unit (IPAddressDeny/IPAddressAllow with the new peers key); the firewall wall compares the loaded table with the file; check timer every 2 minutes. - Units wall asks systemctl show; the file scan covers system.control, system.attached, generators, /run, /usr/lib and prefix drop-ins. - Gateway wall fails when allow or peers cover the gateway. - lockedSettings takes whole objects and record entries; blank organization refused; StartLimit for a policy T3 cannot apply. - run() kills the whole process group on a timeout. Co-Authored-By: Claude Opus 5.5 --- fork-kit/pc-agents/README.md | 70 ++++++--- fork-kit/pc-agents/lib/pc_agents.py | 154 +++++++++++++++--- fork-kit/pc-agents/sbin/pc-agents | 119 ++++++++++---- fork-kit/pc-agents/tests/check.test.py | 198 +++++++++++++++++++++++- fork-kit/pc-agents/tests/fake_distro.py | 59 +++++-- 5 files changed, 505 insertions(+), 95 deletions(-) diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md index b825f0888c34a..c16182d0de384 100644 --- a/fork-kit/pc-agents/README.md +++ b/fork-kit/pc-agents/README.md @@ -5,21 +5,34 @@ T3's user, so the link is not the wall: the distro is. This kit makes a fresh WS T3 in it as `agent`, a user with no sudo, under a root-owned system unit. Everything that holds the wall is root's, so the agent cannot undo it: -| Wall | What the kit does | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` | -| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | -| No way out of the user | one login user (`agent`), in no group but its own; cron, at and ssh masked | -| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | -| No LAN, no Windows host | nftables, for the agent's uid only: private, link-local, CGNAT (Tailscale) and multicast ranges dropped, so the Windows host (the NAT gateway), the home network and the tailnet are out of reach. Public addresses (GitHub, Anthropic, package mirrors) and replies to the linked T3 go through | -| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked | -| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and daily (a failed wall stops T3) | +| Wall | What the kit does | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` and `/run/tailscale` (the Tailscale daemon's socket) | +| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | +| No way out of the user | one login user (`agent`), in no group but its own; the distro's default user is `root`, not `agent`; cron, at, ssh and the agent's systemd user manager (`user@.service`) masked, so no login, linger or user unit of the agent runs anything outside T3's unit | +| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | +| No LAN, no Windows host | twice, nftables for the agent's uid and the unit's `IPAddressDeny=`: private, link-local, CGNAT and multicast ranges blocked but for `allow` and the linking `peers`, so the Windows host, the home network and the tailnet are out of reach; public addresses go through | +| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked; T3 itself is root's (under `/opt/pc-agents/t3`) | +| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and every 2 minutes (a failed wall stops T3; a check that cannot run or crashes stops it too) | `pc-agents check` is the fleet's WSL2 walls check (`.github/scripts/fork-fleet-walls.py` with -`fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: every file above is root's, -mode 644 and exactly what the kit writes, nothing overrides the unit (no drop-in), the masks are in place, `agent` is -the only login user, the kit's code is root's, no Windows drive is mounted, the firewall table is loaded, and the -default gateway (the Windows host) is in a blocked range. The fleet's `mounts`, `sockets` and `userns` walls are +`fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: + +- every file above is root's, mode 644 and exactly what the kit writes (and recorded in the root-owned + `/etc/pc-agents/installed.tsv`); +- nothing overrides the kit's units: no drop-in or other copy in any folder systemd reads (`system.control`, + `system.attached`, the generators, `/run`, `/usr/lib`, prefix drop-ins like `t3code-.service.d`), and + `systemctl show` agrees (the kit's file, no drop-in, no pending reload, T3 and the timer enabled); +- the masks are in place, no user lingers, the agent has no systemd user units; +- `agent` is the only login user; +- the kit's code (`/opt/pc-agents`, the release folder it runs) and T3's runtime are root's, and T3's `t3` is the one + setup installed (its hash is in the record); +- no Windows drive is mounted; +- the loaded firewall table is the one from the kit's `firewall.nft`: `pc-agents load-firewall` saves the file's hash + and what the kernel then lists, and any later change, or an older file left loaded, differs; +- the default gateway (the Windows host) is in a blocked range, and no `allow` or `peers` entry lets it through. + +The fleet's `mounts`, `sockets` and `userns` walls are about the CI launcher's job units, so they are not reported here; the unit's own sandbox is checked instead (it must be the kit's file, byte for byte, with no drop-in). @@ -44,6 +57,7 @@ The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine "memoryMax": "8G", "cpuQuota": "400%", "nameservers": ["1.1.1.1", "9.9.9.9"], + "peers": ["100.101.102.103/32"], "tailscaleTag": "tag:pc-agents" } } @@ -57,31 +71,44 @@ The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine | `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | | `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | | `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | -| `lockedSettings` | more T3 settings to lock, `{path: value}`, one field per dotted path: the child ceiling (`agentSlots.…`) | +| `lockedSettings` | more T3 settings to lock, `{path: value}`: the child ceiling (`agentSlots.…`), see below | | `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | | `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | -| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default | +| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default; never the Windows host | +| `peers` | the linking T3's Tailscale address(es) (CIDR list, 1 to 16): the only private addresses the unit talks to | | `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`, `usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's (`apps/server/src/policy/managedSettings.ts`), and a test proves it for the fork's keys. Lock one field per dotted -path (`agentSlots.maxPerMachine`): T3 takes an object only as a setting's complete value and refuses to start on a -policy it cannot apply, so the kit refuses object values. A field inside a setting that can be empty must be locked -as the whole setting; T3 says so in its log (`journalctl -u t3code-agents`) and does not start. +path (`agentSlots.maxPerMachine`), or give a whole setting (`agentSlots`) or one record entry +(`providerInstances.codex-work`) as an object holding every field: T3 takes an object only as a complete value. The kit +refuses what is cheap to tell (a field of a provider instance, an object deeper than a record entry, the kit's own +two settings, a blank `organization`); T3 refuses the rest, like a field inside a setting that can be empty (lock the +whole setting), says why in its log (`journalctl -u t3code-agents`) and does not start. T3 exits 1 on a policy it +cannot apply, the same code as any other failure, so the unit cannot tell it apart (`RestartPreventExitStatus=`); +it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). ## Limits - **One VM for all distros.** WSL2 runs every distro in one VM: one kernel, one network namespace, one memory cap. The firewall table is therefore global and matches the agent's uid in every distro (keep `agentUid` apart from the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable. - Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. + Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. Another + distro's root can drop or change the global nft table: the check sees it within 2 minutes and stops T3, and the + unit's own `IPAddressDeny=` (per cgroup, not global) holds meanwhile. Abstract unix sockets are shared by every + distro too (the unit cannot hide them; `RestrictAddressFamilies` keeps `AF_UNIX` for T3 itself), so a program in + another distro listening on one is reachable. +- **The policy binds an honest T3:** it is T3 that applies the locks. The kit makes sure the T3 that runs is the one + root installed (root-owned under `/opt/pc-agents/t3`, its hash recorded and checked), not one the agent could + change. An agent can still start its own programs (even its own copy of T3) inside the unit: those ignore the + policy, but stay inside every other wall. - **Public internet is open** to the agent: the firewall blocks private ranges, not the public internet (GitHub's and Anthropic's addresses are many and change). Add an outbound proxy if that matters. - **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not run; the distro is the sandbox, so use full access. - **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, - but the code files are not hashed. + but the code files are not hashed (T3's `t3` is). ## Tests @@ -90,5 +117,6 @@ python3 -I -B fork-kit/pc-agents/tests/check.test.py ``` A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake `nft` and -`systemctl`: every wall passes on a walled distro, and each one fails on its own when its part is taken away. +`systemctl` (it keeps which units are enabled and running): every wall passes on a walled distro, and each one +fails on its own when its part is taken away. `apps/server/src/policy/managedSettings.forkKeys.test.ts` proves the managed policy locks the fork's settings. diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py index 29000cb399cb1..703fbab0a5062 100644 --- a/fork-kit/pc-agents/lib/pc_agents.py +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -2,7 +2,7 @@ Every path is read and written under ROOT, which is '' (the real machine) except in the tests, where PC_AGENTS_ROOT names a fake one; root never gets a fake one. Standard library only.""" -import hashlib, ipaddress, json, os, pwd, re, subprocess +import hashlib, ipaddress, json, os, re, signal, subprocess import machine_file @@ -13,8 +13,11 @@ OWNER = os.geteuid() if ROOT else 0 # who must own every file the kit writes HOME, USER = '/home/agent', 'agent' OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' -RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote +RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote (a path may have two mid-write) +LOADED = '/run/pc-agents/firewall.loaded' # sha256 of the firewall file and `nft list` of it, saved when loaded +T3 = OPT + '/t3' # T3's runtime, installed by root (T3CODE_HOME of scripts/install.sh): the agent cannot change it UNIT, WALLS = 't3code-agents.service', 'pc-agents-walls' +UNITS = (UNIT, WALLS + '.service', WALLS + '.timer') CHECK = OPT + '/current/sbin/pc-agents' WALLS_PY = OPT + '/current/walls/fork-fleet-walls.py' # Walls of fork-fleet-walls/wsl2 that are about the fleet launcher's job units. Here the job unit is T3's own unit, @@ -22,8 +25,12 @@ UNIT_WALLS = {'mounts', 'sockets', 'userns'} MASKS = ['/etc/binfmt.d/WSLInterop.conf'] + ['/etc/systemd/system/' + u for u in ('cron.service', 'atd.service', 'ssh.service', 'ssh.socket')] -UNIT_DIRS = ('/etc/systemd/system', '/run/systemd/system', '/usr/local/lib/systemd/system', '/usr/lib/systemd/system', - '/lib/systemd/system', '/run/systemd/transient', '/run/systemd/generator.early') +# Every folder systemd reads system units and drop-ins from (systemd.unit(5)); the kit's units live in the first. +UNIT_DIRS = ('/etc/systemd/system', '/etc/systemd/system.control', '/run/systemd/system.control', + '/run/systemd/transient', '/run/systemd/generator.early', '/etc/systemd/system.attached', + '/run/systemd/system', '/run/systemd/system.attached', '/run/systemd/generator', + '/usr/local/lib/systemd/system', '/usr/lib/systemd/system', '/lib/systemd/system', + '/run/systemd/generator.late') BLOCKED4 = ('0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '169.254.0.0/16', '172.16.0.0/12', '192.168.0.0/16', '224.0.0.0/4', '240.0.0.0/4') BLOCKED6 = ('fc00::/7', 'fe80::/10', 'ff00::/8') @@ -56,33 +63,60 @@ def is_ip_list(value, net, glob=False, most=64): SPEC = { - 'organization': (str, machine_file.REQUIRED, lambda v: None if 0 < len(v) <= 80 else '1 to 80 characters'), + 'organization': (str, machine_file.REQUIRED, + lambda v: None if v.strip() and len(v) <= 80 else '1 to 80 characters, not blank'), 't3Version': (str, machine_file.REQUIRED, lambda v: None if VERSION.fullmatch(v) else 'an exact version, no v'), 't3ReleaseBaseUrl': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'https://[A-Za-z0-9./_-]+', v) else 'an https URL'), 'port': (int, machine_file.REQUIRED, lambda v: None if 1024 <= v <= 65535 else '1024 to 65535'), 'agentUid': (int, 1100, lambda v: None if 1000 <= v < 60000 else '1000 to 59999'), 'maxRuntimeMode': (str, machine_file.REQUIRED, lambda v: None if v in MODES else 'one of ' + ', '.join(MODES)), - 'lockedSettings': (dict, {}, lambda v: next((f'{k}: set by the kit itself, or not a settings path' for k in v - if k in KIT_SETTINGS or not re.fullmatch(r'[a-z]\w*(\.\w+)*', k)), None) - or next((f'{k}: lock one field per dotted path ({k}.field), not an object (T3 takes an object ' - 'only as the whole setting and does not start on a policy it cannot apply)' - for k, x in v.items() if isinstance(x, dict)), None)), + 'lockedSettings': (dict, {}, lambda v: next(filter(None, (locked_problem(p, x) for p, x in v.items())), None)), 'memoryMax': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*[KMGT]', v) else 'like 8G'), 'cpuQuota': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*%', v) else 'like 400%'), 'nameservers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, False, True, 3)), 'allow': (list, [], lambda v: is_ip_list(v, True)), + 'peers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, True, most=16) if v else '1 to 16 entries'), 'tailscaleTag': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'tag:[a-z0-9-]{1,50}', v) else 'tag:name'), } +def locked_problem(path, value): + """Why T3 would refuse to lock path to value, where that is cheap to tell (T3 checks the rest and does not start + on a policy it cannot apply: managedSettings.ts), or None. An object is a complete value: a whole setting + (`agentSlots`) or one entry of a record (`providerInstances.codex-work`), never a part of one.""" + segments = path.split('.') + if not re.fullmatch(r'[a-z]\w*(\.[A-Za-z0-9_-]+)*', path) or segments[0] in KIT_SETTINGS: + return f'{path}: set by the kit itself, or not a settings path' + if segments[0] == 'providerInstances' and len(segments) > 2: + return f'{path}: lock providerInstances.{segments[1]} as a whole (T3 refuses one field of a provider instance)' + if isinstance(value, dict) and len(segments) > 2: + return (f'{path}: an object only as a whole setting ({segments[0]}) or a record entry; lock one field per ' + 'dotted path instead (T3 does not start on a partial object)') + return None + + def config(): section = machine_file.read_section(KIT, ROOT + machine_file.default_path() if ROOT else None, OWNER) return machine_file.fields(section, SPEC, f'{machine_file.default_path()}: {KIT}') def runtime(c): - return f'{HOME}/.t3/runtime/versions/{c["t3Version"]}/t3' + """T3's version folder (root's, from scripts/install.sh with T3CODE_HOME=T3); the unit runs its `t3`.""" + return f'{T3}/runtime/versions/{c["t3Version"]}' + + +def masks(c): + """Units the kit masks: WSL's binfmt handler, cron, at, ssh and the agent's systemd user manager (with it masked, + no login of the agent, no linger and no user unit runs anything outside T3's unit).""" + return MASKS + [f'/etc/systemd/system/user@{c["agentUid"]}.service'] + + +def dropin_dirs(unit): + """Every drop-in folder name systemd reads for unit: its own, its type's and each dash prefix's.""" + name, kind = unit.rsplit('.', 1) + parts = name.split('-') + return [unit + '.d', kind + '.d'] + ['-'.join(parts[:i]) + f'-.{kind}.d' for i in range(1, len(parts))] def render(c): @@ -104,15 +138,18 @@ def render(c): Description=T3 Code for agents, walled (pc-agents kit) Wants=network-online.target After=network-online.target +# T3 exits 1 on a policy it cannot apply, like on any other failure: give up after 5 starts in 10 minutes. +StartLimitIntervalSec=600 +StartLimitBurst=5 [Service] User={USER} Group={USER} Environment=HOME={HOME} T3CODE_HOME={HOME}/.t3 T3CODE_PORT={c["port"]} T3CODE_LAN=true Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin -ExecStartPre=+/usr/sbin/nft -f {STATE}/firewall.nft +ExecStartPre=+{CHECK} load-firewall ExecStartPre=+{CHECK} check -ExecStart={runtime(c)} serve +ExecStart={runtime(c)}/t3 serve Restart=on-failure RestartSec=30 UMask=0077 @@ -126,7 +163,9 @@ def render(c): PrivateDevices=yes PrivateIPC=yes ProtectProc=invisible -InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl +InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale +IPAddressDeny={" ".join(BLOCKED4 + BLOCKED6)} +IPAddressAllow={" ".join(c["peers"] + c["allow"])} RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS SystemCallArchitectures=native @@ -145,7 +184,7 @@ def render(c): WantedBy=multi-user.target ''' wsl = HEAD + ('[boot]\nsystemd=true\n[automount]\nenabled=false\nmountFsTab=false\n[interop]\nenabled=false\n' - f'appendWindowsPath=false\n[network]\ngenerateResolvConf=false\n[user]\ndefault={USER}\n') + 'appendWindowsPath=false\n[network]\ngenerateResolvConf=false\n[user]\ndefault=root\n') sd = '/etc/systemd/system/' return { '/etc/wsl.conf': (wsl, 0o644), @@ -153,10 +192,14 @@ def render(c): '/etc/t3code/policy.json': (json.dumps(policy, indent=2, sort_keys=True) + '\n', 0o644), STATE + '/firewall.nft': (nft, 0o644), sd + UNIT: (unit, 0o644), - sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents daily walls check\n\n[Service]\nType=oneshot\n' - f'ExecStart={CHECK} check --enforce\n', 0o644), - sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents daily walls check\n\n[Timer]\nOnCalendar=daily\n' - 'Persistent=true\n\n[Install]\nWantedBy=timers.target\n', 0o644), + # A check that does not succeed, even one that cannot start or crashes before it can stop T3, stops T3. + sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\n\n[Service]\nType=oneshot\n' + f'ExecStart={CHECK} check --enforce\n' + 'ExecStopPost=/bin/sh -c \'[ "$$SERVICE_RESULT" = success ] || ' + f'exec /usr/bin/systemctl stop {UNIT}\'\n', 0o644), + sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents walls check every 2 minutes\n\n[Timer]\n' + 'OnActiveSec=1min\nOnUnitActiveSec=2min\nAccuracySec=10s\n\n[Install]\n' + 'WantedBy=timers.target\n', 0o644), } @@ -164,12 +207,58 @@ def sha(text): return hashlib.sha256(text.encode() if isinstance(text, str) else text).hexdigest() +def file_sha(path): + h = hashlib.sha256() + with open(ROOT + path, 'rb') as f: + for block in iter(lambda: f.read(1 << 20), b''): + h.update(block) + return h.hexdigest() + + def record(): + """{path: {sha256, ...}} of the record: one hash per path, two while setup replaces a file (old and new).""" + out = {} try: with open(ROOT + RECORD, encoding='utf-8') as f: - return dict(reversed(line.rstrip('\n').split('\t', 1)) for line in f if '\t' in line) + for line in f: + if '\t' in line: + h, path = line.rstrip('\n').split('\t', 1) + out.setdefault(path, set()).add(h) except FileNotFoundError: - return {} + pass + return out + + +def save(path, text): + """Write a small state file whole (new file, one rename) unless it already says text.""" + real = ROOT + path + try: + with open(real, encoding='utf-8') as f: + if f.read() == text: + return + except (FileNotFoundError, UnicodeDecodeError): + pass + os.makedirs(os.path.dirname(real), mode=0o755, exist_ok=True) + with open(real + '.new', 'w', encoding='utf-8') as f: + f.write(text) + os.replace(real + '.new', real) + + +def nft_listing(): + r = run(['/usr/sbin/nft', 'list', 'table', 'inet', 'pc_agents']) + return r.stdout if r.returncode == 0 else None + + +def load_firewall(): + """Load firewall.nft and save its hash with what the kernel then lists, so the check can tell that the loaded + table is this file's (any change by anyone, or an older file left loaded, differs). Returns why not, or None.""" + body = file_sha(STATE + '/firewall.nft') + r = run(['/usr/sbin/nft', '-f', STATE + '/firewall.nft']) + listing = nft_listing() if r.returncode == 0 else None + if listing is None: + return f'cannot load {STATE}/firewall.nft: {(r.stderr or r.stdout).strip()[-300:]}' + save(LOADED, f'{body}\n{listing}') + return None def lstat(path): @@ -214,6 +303,23 @@ def user_problem(c, need_agent): return f'user {USER} is in other groups ({", ".join(extra)}): it must have none' if extra else None -def run(argv, **kw): - return subprocess.run([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, capture_output=True, text=True, - timeout=120, **kw) +def run(argv, timeout=120): + """Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops, the + whole group is killed (a download's curl too); a timeout returns exit 124 and says so in stderr.""" + p = subprocess.Popen([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + out, err = p.communicate(timeout=timeout) + except BaseException as e: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + out, err = p.communicate(timeout=10) + except subprocess.TimeoutExpired: # something left the group but holds the pipes: stop reading + out, err = '', '' + if not isinstance(e, subprocess.TimeoutExpired): + raise + return subprocess.CompletedProcess(argv, 124, out, f'{err}\n{argv[0]} took more than {timeout} s; stopped it') + return subprocess.CompletedProcess(argv, p.returncode, out, err) diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents index 449635b942ef4..73710c4c1ead4 100755 --- a/fork-kit/pc-agents/sbin/pc-agents +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -2,14 +2,16 @@ """pc-agents: wall a WSL2 distro in, then run T3 Code in it as an unprivileged user (fork-kit/pc-agents/README.md). pc-agents check [--enforce] every wall, PASS or FAIL; exit 0 only when all hold. --enforce also - stops T3 when one fails (the daily timer). T3's unit runs it before - every start, so T3 never starts on a failed wall. + stops T3 when one fails (the timer, every 2 minutes). T3's unit runs + it before every start, so T3 never starts on a failed wall. + pc-agents load-firewall as root: load the agent firewall and note what the kernel lists Standard library only.""" import os, re, stat, sys HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -COMMON = os.path.join(os.path.dirname(HERE), 'common') -sys.path[:0] = [os.path.join(HERE, 'lib')] + ([COMMON] if os.path.isdir(COMMON) else []) +LIB = os.path.join(HERE, 'lib') +COMMON = os.path.join(os.path.dirname(HERE), 'common') # the repository's; an installed kit has its own copy in LIB +sys.path[:0] = [LIB] + ([] if os.path.exists(os.path.join(LIB, 'machine_file.py')) else [COMMON]) sys.dont_write_bytecode = True import pc_agents as k # noqa: E402 @@ -18,47 +20,91 @@ def say(text): print(f'pc-agents: {text}', flush=True) -def own_walls(c): - """(wall, what, why or None) for the walls this kit adds to fork-fleet-walls/wsl2.""" +def tree(top, inside=None): + """Why something in the folder top (itself included) is not root's alone, or a link out of inside; or None.""" + bad = [why] if (why := k.safe(top, k.lstat(top), stat.S_ISDIR)) else [] + for here, dirs, names in os.walk(k.ROOT + top): + for n in dirs + names: + p = os.path.join(here, n)[len(k.ROOT):] + st = k.lstat(p) + if stat.S_ISLNK(st.st_mode) and inside and st.st_uid == k.OWNER and \ + os.path.realpath(k.ROOT + p).startswith(os.path.realpath(k.ROOT + inside) + '/'): + continue + bad += [why] if (why := k.safe(p, st, lambda m: stat.S_ISDIR(m) or stat.S_ISREG(m))) else [] + return bad[:3] and '; '.join(bad[:3]) + + +def own_walls(c, before_enable=False): + """(wall, what, why or None) for the walls this kit adds to fork-fleet-walls/wsl2. before_enable: setup asks + before it enables T3 and the timer, so they need not be enabled yet.""" out, done = [], k.record() for path, (text, mode) in k.render(c).items(): st = k.lstat(path) why = k.safe(path, st, stat.S_ISREG) if why is None: with open(k.ROOT + path, 'rb') as f: - why = None if f.read() == text.encode() and done.get(path) == k.sha(text) else \ + why = None if f.read() == text.encode() and k.sha(text) in done.get(path, ()) else \ f'{path} differs from what the kit writes (pc-agents setup)' out.append(('file:' + os.path.basename(path), f'{path} is the kit\'s, root-owned', why)) - masks = [p for p in k.MASKS if not (k.lstat(p) and os.path.islink(k.ROOT + p) - and os.readlink(k.ROOT + p) == '/dev/null')] - out.append(('masks', 'WSL interop, cron, at and ssh are masked', masks and 'not masked: ' + ', '.join(masks))) - extra = [f'{d}/{n}' for d in k.UNIT_DIRS for n in (k.UNIT + '.d', k.WALLS + '.service.d') + ( - ('service.d',) if d in k.UNIT_DIRS[:2] else ()) + ((k.UNIT,) if d != k.UNIT_DIRS[0] else ()) - if k.lstat(f'{d}/{n}')] - out.append(('drop-ins', 'nothing overrides the unit', extra and 'found ' + ', '.join(extra))) + out.append(('record', f'{k.RECORD} is root-owned', k.safe(k.RECORD, k.lstat(k.RECORD), stat.S_ISREG))) + masks = [p for p in k.masks(c) if not (k.lstat(p) and os.path.islink(k.ROOT + p) + and os.readlink(k.ROOT + p) == '/dev/null')] + out.append(('masks', 'WSL interop, cron, at, ssh and the agent\'s user manager are masked', + masks and 'not masked: ' + ', '.join(masks))) + linger = sorted(os.listdir(k.ROOT + '/var/lib/systemd/linger')) if k.lstat('/var/lib/systemd/linger') else [] + out.append(('linger', 'no user lingers', linger and 'lingering: ' + ', '.join(linger[:5]))) + units = [p for p in ('.config/systemd/user', '.config/systemd/user.control', '.local/share/systemd/user') + if os.path.isdir(f'{k.ROOT}{k.HOME}/{p}') and os.listdir(f'{k.ROOT}{k.HOME}/{p}')] + out.append(('user-units', f'{k.USER} has no systemd user units', units and 'found ~/' + ', ~/'.join(units))) + extra = [f'{d}/{n}' for d in k.UNIT_DIRS for u in k.UNITS + for n in k.dropin_dirs(u) + ([u] if d != k.UNIT_DIRS[0] else []) if k.lstat(f'{d}/{n}')] + out.append(('drop-ins', 'no drop-in or other copy of the kit\'s units', extra and 'found ' + ', '.join(extra[:5]))) + seen = [] + for u in k.UNITS: # what systemd has loaded, not only what is on disk + r = k.run(['/usr/bin/systemctl', 'show', '-p', + 'LoadState,FragmentPath,DropInPaths,UnitFileState,NeedDaemonReload', u]) + got = dict(line.split('=', 1) for line in r.stdout.splitlines() if '=' in line) + want = {'LoadState': 'loaded', 'FragmentPath': f'{k.UNIT_DIRS[0]}/{u}', 'DropInPaths': '', + 'NeedDaemonReload': 'no'} + if u != k.WALLS + '.service' and not before_enable: + want['UnitFileState'] = 'enabled' + seen += [f'{u}: {key}={got.get(key)!r}' for key, v in want.items() if r.returncode or got.get(key) != v] + out.append(('units', 'systemd runs the kit\'s units as written, enabled', seen and '; '.join(seen[:3]))) out.append(('agent', f'one login user, {k.USER}, in no group but its own', k.user_problem(c, True))) - code = [] - for top, dirs, names in os.walk(k.ROOT + k.OPT + '/current/'): - for n in dirs + names: - p = os.path.join(top, n)[len(k.ROOT):] - code += [why] if (why := k.safe(p, k.lstat(p), lambda m: stat.S_ISDIR(m) or stat.S_ISREG(m))) else [] current = k.lstat(k.OPT + '/current') - code += [] if current and current.st_uid == k.OWNER else [f'{k.OPT}/current is not the kit\'s'] - out.append(('kit-code', f'{k.OPT}/current is root-owned', code[:3] and '; '.join(code[:3]))) + target = os.readlink(k.ROOT + k.OPT + '/current') if current and stat.S_ISLNK(current.st_mode) else '' + code = [] if current and current.st_uid == k.OWNER and re.fullmatch(r'[0-9a-f]{40}', target) else \ + [f'{k.OPT}/current is not the kit\'s link to a release folder'] + code += [why for d in ('/', '/opt', k.OPT) if (why := k.safe(d, k.lstat(d), stat.S_ISDIR))] + code += [why] if target and (why := tree(f'{k.OPT}/{target}')) else [] + out.append(('kit-code', f'{k.OPT} and the release it runs are root\'s alone', code[:3] and '; '.join(code[:3]))) + folder, rt = k.runtime(c), [] + if k.lstat(folder): # else not installed (yet): T3 cannot start + rt = [why for d in (k.T3, k.T3 + '/runtime', k.T3 + '/runtime/versions') + if (why := k.safe(d, k.lstat(d), stat.S_ISDIR))] or [why for why in [tree(folder, folder)] if why] + if not rt and k.lstat(folder + '/t3') and k.file_sha(folder + '/t3') not in done.get(folder + '/t3', ()): + rt = [f'{folder}/t3 is not the one pc-agents setup installed'] + out.append(('runtime', f'T3 {c["t3Version"]} is root\'s alone, as installed', rt and '; '.join(rt))) mounts = [] try: with open(k.ROOT + '/proc/self/mountinfo', encoding='utf-8', errors='replace') as f: for line in f: fields = line.split(' - ') where, fstype = fields[0].split()[4], (fields[1].split() or [''])[0] if len(fields) > 1 else '' - if fstype in ('9p', 'drvfs', 'virtiofs') and not where.startswith('/usr/lib/wsl'): + if fstype in ('9p', 'drvfs', 'virtiofs') and not (where + '/').startswith('/usr/lib/wsl/'): mounts.append(f'{where} ({fstype})') except (OSError, IndexError) as e: mounts.append(f'cannot read /proc/self/mountinfo ({e})') out.append(('automount', 'no Windows drive is mounted', mounts and 'mounted: ' + ', '.join(mounts[:3]))) - r = k.run(['/usr/sbin/nft', 'list', 'table', 'inet', 'pc_agents']) - nft_ok = r.returncode == 0 and f'skuid != {c["agentUid"]}' in r.stdout and ' drop' in r.stdout - out.append(('firewall', 'the agent firewall is loaded', None if nft_ok else 'table inet pc_agents is not loaded')) + saved = None + if k.safe(k.LOADED, k.lstat(k.LOADED), stat.S_ISREG) is None: + with open(k.ROOT + k.LOADED, encoding='utf-8', errors='replace') as f: + saved = f.read() + listing = k.nft_listing() + fw = 'table inet pc_agents is not loaded' if listing is None else None if saved == \ + f'{k.sha(k.render(c)[k.STATE + "/firewall.nft"][0])}\n{listing}' else \ + 'the loaded table inet pc_agents is not the one from the kit\'s firewall.nft (pc-agents load-firewall)' + out.append(('firewall', 'the kit\'s agent firewall is what is loaded', fw)) gateway = None try: with open(k.ROOT + '/proc/net/route', encoding='utf-8') as f: @@ -66,10 +112,13 @@ def own_walls(c): gateway = row and k.ipaddress.ip_address(bytes.fromhex(row[2])[::-1]) except (OSError, ValueError) as e: gateway = f'unknown ({e})' - blocked = isinstance(gateway, k.ipaddress.IPv4Address) and \ - any(gateway in k.ipaddress.ip_network(n) for n in k.BLOCKED4) - out.append(('gateway', 'the Windows host (the default gateway) is in a blocked range', - None if blocked else f'the gateway is {gateway}, not in a blocked range')) + + def nets(names): + return [n for n in names if isinstance(gateway, k.ipaddress.IPv4Address) and gateway in k.ipaddress.ip_network(n)] + opened = nets(c['allow'] + c['peers']) + out.append(('gateway', 'the Windows host (the default gateway) is in a blocked range and not let through', + f'allow or peers let the gateway {gateway} through: {", ".join(opened)}' if opened else + None if nets(k.BLOCKED4) else f'the gateway is {gateway}, not in a blocked range')) return out @@ -89,24 +138,28 @@ def wsl2_walls(): return [(f'wsl2:{w}', what, None if mark == 'PASS' else what) for mark, w, what in lines if w not in k.UNIT_WALLS] -def check(enforce): +def check(enforce, before_enable=False): try: c = k.config() - results = wsl2_walls() + own_walls(c) + results = wsl2_walls() + own_walls(c, before_enable) except Exception as e: # a wall that cannot be told does not hold results = [('check', 'the walls can be read', f'{e.__class__.__name__}: {e}')] for wall, what, why in results: print(f'FAIL {wall} {what}: {why}'[:500] if why else f'PASS {wall} {what}', flush=True) failed = any(why for _, _, why in results) if failed and enforce: - k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) - say(f'a wall does not hold: stopped {k.UNIT}') + r = k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) + say(f'a wall does not hold: stopped {k.UNIT}' if r.returncode == 0 else + f'a wall does not hold, and stopping {k.UNIT} FAILED: {r.stderr.strip()[-300:]}') return 1 if failed else 0 def main(argv): if argv[:1] == ['check'] and argv[1:] in ([], ['--enforce']): return check(argv[1:] == ['--enforce']) + if argv == ['load-firewall']: + why = k.load_firewall() + return say(why) or 1 if why else 0 print(__doc__, file=sys.stderr) return 2 diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py index 03fc4bb708724..1f69c0b77b80b 100644 --- a/fork-kit/pc-agents/tests/check.test.py +++ b/fork-kit/pc-agents/tests/check.test.py @@ -3,10 +3,10 @@ python3 -I -B fork-kit/pc-agents/tests/check.test.py """ -import os, shutil, sys, unittest +import os, shutil, subprocess, sys, time, unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from fake_distro import Case, DRVFS, SHA1, k # noqa: E402 +from fake_distro import CONFIG, Case, DRVFS, SHA1, k # noqa: E402 class Walls(Case): @@ -25,8 +25,9 @@ def test_all_hold(self): self.assertEqual(code, 0, out) walls = [line.split()[1] for line in out.splitlines()] for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', - 'drop-ins', 'agent', 'kit-code', 'automount', 'firewall', 'gateway', 'file:policy.json', - 'file:t3code-agents.service', 'file:firewall.nft', 'file:wsl.conf'): + 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'kit-code', 'runtime', + 'automount', 'firewall', 'gateway', 'file:policy.json', 'file:t3code-agents.service', + 'file:firewall.nft', 'file:wsl.conf', 'file:pc-agents-walls.timer'): self.assertIn(wall, walls) for wall in k.UNIT_WALLS: # the launcher's job units: not T3's unit self.assertNotIn('wsl2:' + wall, walls) @@ -36,6 +37,15 @@ def test_enforce_stops_t3(self): code, out = self.check('--enforce') self.assertEqual(code, 1, out) self.assertIn('stop t3code-agents.service', self.log()) + self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) + self.assertIn('stopped t3code-agents.service', out) + + def test_enforce_says_when_the_stop_fails(self): + os.unlink(self.path('/log/nft-loaded')) + self.write('/log/systemctl-fail', '') + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('stopping t3code-agents.service FAILED', out) def test_each_wall(self): def edit(p, old, new): @@ -55,6 +65,12 @@ def edit(p, old, new): 'wsl2:interop': lambda: self.write('/proc/sys/fs/binfmt_misc/WSLInterop', 'enabled\ninterpreter /init\nflags: PF\noffset 0\nmagic 4d5a\n'), 'wsl2:networking': edit('/usr/bin/wslinfo', 'echo nat', 'echo mirrored'), + 'record': lambda: os.chmod(self.path(k.RECORD), 0o666), + 'linger': lambda: self.write('/var/lib/systemd/linger/agent', ''), + 'user-units': lambda: self.write('/home/agent/.config/systemd/user/x.service', '[Service]\n'), + 'units': lambda: self.write('/log/units/t3code-agents.service.dropins', '/run/systemd/system/x.conf'), + 'runtime': lambda: os.chmod(self.path(k.runtime(CONFIG) + '/t3'), 0o777), + 'file:pc-agents-walls.timer': edit('/etc/systemd/system/pc-agents-walls.timer', '2min', '1d'), } clean = {p: v[4] for p, v in self.snapshot().items()} for wall, change in cases.items(): @@ -65,6 +81,78 @@ def edit(p, old, new): change() self.fails(wall, *(['wsl2:resolv'] if wall == 'file:resolv.conf' else [])) + def again(self, **section): + """machine.json changed and set up again: the new files, recorded, their firewall loaded.""" + self.machine({**CONFIG, **section}) + self.files() + self.assertIsNone(k.load_firewall()) + + def test_agent_user_manager_masked(self): + os.unlink(self.path('/etc/systemd/system/user@1100.service')) + self.fails('masks') + + def test_unit_copies_and_prefix_drop_ins(self): + for p in ('/usr/lib/systemd/system/t3code-agents.service', '/run/systemd/generator.late/pc-agents-walls.timer', + '/etc/systemd/system/t3code-.service.d/', '/etc/systemd/system/pc-agents-.timer.d/', + '/run/systemd/system.attached/t3code-agents.service.d/', '/etc/systemd/system/service.d/'): + with self.subTest(path=p): + self.write(p, '[Service]\n') + self.fails('drop-ins') + (os.rmdir if p.endswith('/') else os.unlink)(self.path(p.rstrip('/'))) + + def test_units_as_systemd_loaded_them(self): + for name, change in (('timer disabled', lambda: os.unlink(self.path('/log/units/pc-agents-walls.timer.enabled'))), + ('T3 disabled', lambda: os.unlink(self.path('/log/units/t3code-agents.service.enabled'))), + ('not reloaded', lambda: self.write('/log/units/reload', '')), + ('drop-in', lambda: self.write('/log/units/pc-agents-walls.service.dropins', '/x.conf'))): + with self.subTest(name): + shutil.rmtree(self.root) + self.setUp() + change() + self.fails('units') + + def test_firewall_changed_after_it_was_loaded(self): + self.write('/log/nft-loaded', self.read('/log/nft-loaded').replace('accept', 'accept\n', 1)) + self.fails('firewall') + + def test_firewall_file_newer_than_the_loaded_table(self): + self.machine({**CONFIG, 'allow': ['192.168.1.0/24']}) + self.files() + self.fails('firewall') + self.assertIsNone(k.load_firewall()) + self.assertEqual(self.check()[0], 0) + + def test_allow_or_peers_covering_the_gateway(self): + for key in ('allow', 'peers'): + with self.subTest(key): + self.again(**{key: ['172.16.0.0/12']}) + self.fails('gateway') + self.again() + self.assertEqual(self.check()[0], 0) + + def test_runtime_changed_or_missing(self): + t3 = k.runtime(CONFIG) + '/t3' + self.write(t3, {'text': '#!/bin/sh\nexec /home/agent/x\n', 'mode': '755'}) + self.fails('runtime') + os.chmod(self.path('/opt/pc-agents/t3/runtime'), 0o777) + self.fails('runtime') + shutil.rmtree(self.path(k.runtime(CONFIG))) # not installed: T3 cannot start, nothing to wall + os.chmod(self.path('/opt/pc-agents/t3/runtime'), 0o755) + code, out = self.check() + self.assertEqual(code, 0, out) + + def test_kit_code_folders(self): + os.chmod(self.path('/opt/pc-agents'), 0o775) + self.fails('kit-code') + os.chmod(self.path('/opt/pc-agents'), 0o755) + os.symlink('/etc', self.path(f'/opt/pc-agents/{SHA1}/lib/etc')) + self.fails('kit-code') + + def test_automount_only_under_usr_lib_wsl(self): + self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo') + + '45 29 0:45 / /usr/lib/wslfoo rw - 9p drvfs rw\n') + self.fails('automount') + def test_no_config_fails(self): os.unlink(self.path('/etc/fork-kit/machine.json')) code, out = self.check() @@ -72,5 +160,107 @@ def test_no_config_fails(self): self.assertIn('FAIL check ', out) +class Holes(Case): + """Ways the agent could run code outside T3's walled unit, or the walls be changed under systemd.""" + def setUp(self): + super().setUp() + self.walled() + + def test_agent_user_unit(self): + self.write('/home/agent/.config/systemd/user/x.service', '[Service]\nExecStart=/home/agent/x\n') + os.makedirs(self.path('/home/agent/.config/systemd/user/default.target.wants')) + os.symlink('../x.service', self.path('/home/agent/.config/systemd/user/default.target.wants/x.service')) + self.write('/var/lib/systemd/linger/agent', '') + self.assertEqual(self.check()[0], 1) + + def test_system_control_drop_in(self): + self.write('/etc/systemd/system.control/t3code-agents.service.d/50-x.conf', '[Service]\nRestrictAddressFamilies=\n') + self.assertEqual(self.check()[0], 1) + + def test_timer_drop_in(self): + self.write('/etc/systemd/system/pc-agents-walls.timer.d/off.conf', '[Timer]\nOnCalendar=\n') + self.assertEqual(self.check()[0], 1) + + def test_prefix_drop_in(self): + self.write('/etc/systemd/system/t3code-.service.d/x.conf', '[Service]\nUser=root\n') + self.assertEqual(self.check()[0], 1) + + +class Crash(Case): + """A check that cannot run, or crashes, still stops T3 (the walls service's ExecStopPost).""" + def setUp(self): + super().setUp() + self.walled() + + def stop_post(self, result): + line = next(x for x in k.render(k.config())['/etc/systemd/system/pc-agents-walls.service'][0].splitlines() + if x.startswith('ExecStopPost=')) + cmd = line.split('=', 1)[1].replace('$$', '$').replace('/usr/bin/systemctl', self.path('/usr/bin/systemctl')) + subprocess.run(cmd, shell=True, check=True, env={'SERVICE_RESULT': result, 'PATH': '/usr/bin:/bin'}) + + def test_missing_lib_stops_t3(self): + os.unlink(self.path(f'/opt/pc-agents/{SHA1}/lib/pc_agents.py')) + p = subprocess.run([sys.executable, '-I', self.path(f'/opt/pc-agents/{SHA1}/sbin/pc-agents'), 'check', + '--enforce'], env={'PC_AGENTS_ROOT': self.root, 'PATH': '/usr/bin:/bin'}, + capture_output=True, text=True, timeout=60) + self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertNotIn('stop t3code-agents.service', self.log()) # the check itself could not stop T3 + self.stop_post('exit-code') + self.assertIn('stop t3code-agents.service', self.log()) + self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) + + def test_success_does_not_stop_t3(self): + self.stop_post('success') + self.assertEqual(self.log(), []) + + +class Unit(Case): + """What the kit's files say: things no wall can see from outside the unit.""" + def test_unit(self): + files = k.render(k.config()) + unit = files['/etc/systemd/system/t3code-agents.service'][0] + self.assertIn('\nExecStart=/opt/pc-agents/t3/runtime/versions/1.2.3/t3 serve\n', unit) + self.assertIn(' -/run/tailscale', unit) + deny = next(x for x in unit.splitlines() if x.startswith('IPAddressDeny=')).split('=', 1)[1].split() + self.assertEqual(set(deny), set(k.BLOCKED4 + k.BLOCKED6)) + self.assertIn('\nIPAddressAllow=100.101.102.103/32\n', unit) + self.assertIn('\nStartLimitBurst=5\n', unit) + self.assertIn('\n[user]\ndefault=root\n', files['/etc/wsl.conf'][0]) + self.assertIn('\nOnUnitActiveSec=2min\n', files['/etc/systemd/system/pc-agents-walls.timer'][0]) + + def test_config(self): + # The kit passes these; T3 itself tells a complete object from a partial one and does not start on a partial. + ok = {'agentSlots': {'maxPerMachine': 2}, 'providerInstances.codex-work': {'driver': 'codex'}, + 'agentSlots.maxPerMachine': 2, 'providerInstances': {}} + self.assertEqual([p for p, v in ok.items() if k.locked_problem(p, v)], []) + bad = {'providerInstances.codex-work.enabled': True, 'agentSlots.a.b': {'x': 1}, 'automaticUpdates': True, + 'linkedEnvironmentsEnabled.x': 1, 'Agent': 1, 'a..b': 1} + self.assertEqual([p for p, v in bad.items() if not k.locked_problem(p, v)], []) + for section in ({**CONFIG, 'organization': ' '}, {**CONFIG, 'peers': []}, + {key: v for key, v in CONFIG.items() if key != 'peers'}): + with self.subTest(section=section): + self.machine(section) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL check ', out) + + +class Run(Case): + def test_timeout_kills_the_whole_group(self): + self.write('/x', {'text': f'#!/bin/sh\nsleep 60 &\necho $! >{self.root}/pid\nwait\n', 'mode': '755'}) + r = k.run(['/x'], timeout=1) + self.assertEqual(r.returncode, 124) + self.assertIn('took more than 1 s', r.stderr) + pid = int(self.read('/pid')) + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail(f'sleep {pid} still runs') + + if __name__ == '__main__': unittest.main() diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py index abe99cb8ce8ea..79f79362ac740 100644 --- a/fork-kit/pc-agents/tests/fake_distro.py +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -1,6 +1,7 @@ """A fake WSL2 distro in a temporary folder for the pc-agents tests: every fixed path under PC_AGENTS_ROOT, the -current user as root, fake useradd, runuser, systemctl and nft (each logs to /log in the fake root). No root, no -systemd, no network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" +current user as root, fake useradd, env (T3's installer), systemctl, systemd-run and nft (each logs to /log in the +fake root; systemctl keeps which units are enabled and which ExecStart runs in /log/units). No root, no systemd, no +network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" import contextlib, importlib.machinery, importlib.util, io, json, os, shutil, stat, sys, tempfile, unittest KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -9,15 +10,36 @@ SHA1, SHA2 = 'a' * 40, 'b' * 40 CONFIG = {'organization': 'Example lab', 't3Version': '1.2.3', 't3ReleaseBaseUrl': 'https://example.com/releases', 'port': 3800, 'maxRuntimeMode': 'full-access', 'lockedSettings': {'agentSlots.maxPerMachine': 2}, - 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'tailscaleTag': 'tag:pc-agents'} + 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'peers': ['100.101.102.103/32'], + 'tailscaleTag': 'tag:pc-agents'} TOOLS = { # fake programs, under the fake root; each finds the root from its own path 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; prev=; for a; do [ "$prev" = --uid ] && uid=$a; prev=$a; ' 'done\necho "$a:x:$uid:$uid::/home/$a:/bin/bash" >>"$R/etc/passwd"\n' 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', - 'usr/sbin/runuser': 'R="${0%/usr/sbin/runuser}"; echo "$*" >>"$R/log/runuser"\n[ -e "$R/log/runuser-fail" ] && exit 1\n' - 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' - 'd="$R/home/agent/.t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n" >"$d/t3"\n', - 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"\n', + # T3's installer, run by root through env -i (scripts/install.sh with T3CODE_HOME=/opt/pc-agents/t3) + 'usr/bin/env': 'R="${0%/usr/bin/env}"; echo "$*" >>"$R/log/install"\n[ -e "$R/log/install-fail" ] && exit 1\n' + 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' + 'd="$R/opt/pc-agents/t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n# t3 $v\\n" >"$d/t3"\n' + 'chmod 755 "$d/t3"\n', + 'usr/bin/systemd-run': 'R="${0%/usr/bin/systemd-run}"; printf "%s\\n" "$@" >"$R/log/systemd-run"\n', + 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"; S="$R/log/units"; mkdir -p "$S"\n' + '[ -e "$R/log/systemctl-fail" ] && { echo "systemctl: failed (test)" >&2; exit 1; }\n' + 'run() { grep "^ExecStart=" "$R/etc/systemd/system/$1" >"$S/$1.active" 2>/dev/null || :; }\n' + 'case "$1" in\n' + 'enable|disable) v=$1; shift; now=; [ "$1" = --now ] && { now=1; shift; }\n' + ' for u; do if [ $v = enable ]; then : >"$S/$u.enabled"; [ -n "$now" ] && ' + '{ [ -e "$S/$u.active" ] || run "$u"; }\n' + ' else rm -f "$S/$u.enabled"; [ -z "$now" ] || rm -f "$S/$u.active"; fi; done;;\n' + 'restart) shift; for u; do run "$u"; done;;\n' + 'stop) shift; for u; do rm -f "$S/$u.active"; done;;\n' + 'is-active) [ -e "$S/$2.active" ] && echo active || { echo inactive; exit 3; };;\n' + 'show) u=$4; f=; [ -e "$R/etc/systemd/system/$u" ] && f=/etc/systemd/system/$u\n' + ' st=disabled; [ -e "$S/$u.enabled" ] && st=enabled\n' + ' [ "$u" = pc-agents-walls.service ] && st=static\n' + ' echo "LoadState=${f:+loaded}"; echo "FragmentPath=$f"\n' + ' echo "DropInPaths=$(cat "$S/$u.dropins" 2>/dev/null)"; echo "UnitFileState=$st"\n' + ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no;;\n' + 'esac\n', 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', } @@ -119,13 +141,22 @@ def ready(self): code, out = self.setup_run() self.assertEqual(code, 0, out) - def walled(self, sha=SHA1): - """What `pc-agents setup` leaves after `wsl --terminate` and a second run, written here directly.""" - files = k.render(k.config()) + def files(self): + """The kit's files from machine.json, T3's runtime and the record of both, as setup writes them.""" + c = k.config() + files = k.render(c) for p, (text, mode) in files.items(): self.write(p, {'text': text, 'mode': f'{mode:o}'}) - self.write(k.RECORD, ''.join(f'{k.sha(t)}\t{p}\n' for p, (t, _) in sorted(files.items()))) - for p in k.MASKS: + t3 = k.runtime(c) + '/t3' + self.write(t3, {'text': '#!/bin/sh\n', 'mode': '755'}) + self.write(k.RECORD, ''.join(f'{k.sha(t)}\t{p}\n' for p, (t, _) in sorted(files.items())) + + f'{k.file_sha(t3)}\t{t3}\n') + return c + + def walled(self, sha=SHA1): + """What `pc-agents setup` leaves after `wsl --terminate` and a second run, written here directly.""" + c = self.files() + for p in k.masks(c): os.makedirs(os.path.dirname(self.path(p)), exist_ok=True) os.symlink('/dev/null', self.path(p)) with open(self.path('/etc/passwd'), 'a') as f: @@ -136,7 +167,9 @@ def walled(self, sha=SHA1): with open(os.path.join(REPO, src), encoding='utf-8') as f: self.write(f'/opt/pc-agents/{sha}/{dst}', {'text': f.read(), 'mode': f'{mode:o}'}) os.symlink(sha, self.path('/opt/pc-agents/current')) - shutil.copy(self.path(k.STATE + '/firewall.nft'), self.path('/log/nft-loaded')) + self.assertIsNone(k.load_firewall()) + k.run(['/usr/bin/systemctl', 'enable', '--now', k.UNIT, k.WALLS + '.timer']) + os.unlink(self.path('/log/systemctl')) self.terminate() def snapshot(self): From b6ae1ea9bc3619ab51d49e1425a07428f9477c9b Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:44:55 +0000 Subject: [PATCH 3/8] fork-kit: pc-agents fix round 2 (walls) Loopback only to T3's port and a devPorts range (unit SocketBindDeny/Allow, nft drop on lo); walls service ordered after T3 with a start timeout; agent login shell nologin, sudoers and running-timer walls; README limits (hairpin NAT, D-Bus, shm, sock_diag, resolved stub, OSC 52) and how to restart T3 after a wall stopped it. Co-Authored-By: Claude Opus 5.5 --- fork-kit/pc-agents/README.md | 78 ++++++++++++++++--------- fork-kit/pc-agents/lib/pc_agents.py | 63 ++++++++++++++++++-- fork-kit/pc-agents/sbin/pc-agents | 13 ++++- fork-kit/pc-agents/tests/check.test.py | 65 ++++++++++++++++++++- fork-kit/pc-agents/tests/fake_distro.py | 10 ++-- 5 files changed, 190 insertions(+), 39 deletions(-) diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md index c16182d0de384..73213ec8991d4 100644 --- a/fork-kit/pc-agents/README.md +++ b/fork-kit/pc-agents/README.md @@ -5,15 +5,16 @@ T3's user, so the link is not the wall: the distro is. This kit makes a fresh WS T3 in it as `agent`, a user with no sudo, under a root-owned system unit. Everything that holds the wall is root's, so the agent cannot undo it: -| Wall | What the kit does | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` and `/run/tailscale` (the Tailscale daemon's socket) | -| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | -| No way out of the user | one login user (`agent`), in no group but its own; the distro's default user is `root`, not `agent`; cron, at, ssh and the agent's systemd user manager (`user@.service`) masked, so no login, linger or user unit of the agent runs anything outside T3's unit | -| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | -| No LAN, no Windows host | twice, nftables for the agent's uid and the unit's `IPAddressDeny=`: private, link-local, CGNAT and multicast ranges blocked but for `allow` and the linking `peers`, so the Windows host, the home network and the tailnet are out of reach; public addresses go through | -| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked; T3 itself is root's (under `/opt/pc-agents/t3`) | -| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and every 2 minutes (a failed wall stops T3; a check that cannot run or crashes stops it too) | +| Wall | What the kit does | +| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` and `/run/tailscale` (the Tailscale daemon's socket) | +| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | +| No way out of the user | one login user (`agent`), in no group but its own, no sudo rule, login shell `/usr/sbin/nologin`; the distro's default user is `root`, not `agent`; cron, at, ssh and the agent's systemd user manager (`user@.service`) masked, so no login, linger or user unit of the agent runs anything outside T3's unit | +| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | +| No LAN, no Windows host | twice, nftables for the agent's uid and the unit's `IPAddressDeny=`: private, link-local, CGNAT and multicast ranges blocked but for `allow` and the linking `peers`, so the private addresses of the Windows host, the home network and the tailnet are out of reach; public addresses go through (the home's own public address too: see Limits) | +| Localhost only for T3 | the unit's `SocketBindDeny=any` with `SocketBindAllow=` T3's port and `devPorts`, and nftables dropping the agent's other localhost traffic: the agent listens on, and reaches on `localhost`, only T3's port and `devPorts`, so servers of other distros on `localhost` are out of reach | +| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked; T3 itself is root's (under `/opt/pc-agents/t3`) | +| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and every 2 minutes (a failed wall stops T3; a check that cannot run or crashes stops it too) | `pc-agents check` is the fleet's WSL2 walls check (`.github/scripts/fork-fleet-walls.py` with `fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: @@ -23,8 +24,9 @@ so the agent cannot undo it: - nothing overrides the kit's units: no drop-in or other copy in any folder systemd reads (`system.control`, `system.attached`, the generators, `/run`, `/usr/lib`, prefix drop-ins like `t3code-.service.d`), and `systemctl show` agrees (the kit's file, no drop-in, no pending reload, T3 and the timer enabled); -- the masks are in place, no user lingers, the agent has no systemd user units; -- `agent` is the only login user; +- the masks are in place, no user lingers, the agent has no systemd user units, the timer is running; +- `agent` is the only login user, its login shell is `/usr/sbin/nologin`, and no sudo rule (`/etc/sudoers`, + `/etc/sudoers.d`) names it, its group or uid, or every user; - the kit's code (`/opt/pc-agents`, the release folder it runs) and T3's runtime are root's, and T3's `t3` is the one setup installed (its hash is in the record); - no Windows drive is mounted; @@ -63,20 +65,21 @@ The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine } ``` -| Key | What | -| ----------------------- | ----------------------------------------------------------------------------------------------------------- | -| `organization` | the name the managed policy shows in T3's settings | -| `t3Version` | the exact T3 version to run (the same as the T3 that links to this PC); installed with `scripts/install.sh` | -| `t3ReleaseBaseUrl` | where that version's release files are (`T3CODE_RELEASE_BASE_URL` of `scripts/install.sh`) | -| `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | -| `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | -| `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | -| `lockedSettings` | more T3 settings to lock, `{path: value}`: the child ceiling (`agentSlots.…`), see below | -| `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | -| `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | -| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default; never the Windows host | -| `peers` | the linking T3's Tailscale address(es) (CIDR list, 1 to 16): the only private addresses the unit talks to | -| `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | +| Key | What | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `organization` | the name the managed policy shows in T3's settings | +| `t3Version` | the exact T3 version to run (the same as the T3 that links to this PC); installed with `scripts/install.sh` | +| `t3ReleaseBaseUrl` | where that version's release files are (`T3CODE_RELEASE_BASE_URL` of `scripts/install.sh`) | +| `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | +| `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | +| `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | +| `lockedSettings` | more T3 settings to lock, `{path: value}`: the child ceiling (`agentSlots.…`), see below | +| `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | +| `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | +| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default; never the Windows host | +| `devPorts` | optional: the TCP ports the agent's dev servers may listen on and be reached on, `LOW-HIGH` (default `4100-4199`) | +| `peers` | the linking T3's Tailscale address(es) (CIDR list, 1 to 16): the only private addresses the unit talks to | +| `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`, `usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's @@ -89,11 +92,19 @@ whole setting), says why in its log (`journalctl -u t3code-agents`) and does not cannot apply, the same code as any other failure, so the unit cannot tell it apart (`RestartPreventExitStatus=`); it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). +A wall that fails stops T3, and nothing starts it again by itself: fix what `pc-agents check` names (or run +`pc-agents setup` again), then start it as root: `systemctl start t3code-agents.service` (its own start runs the +check first and refuses on a failed wall). + ## Limits - **One VM for all distros.** WSL2 runs every distro in one VM: one kernel, one network namespace, one memory cap. The firewall table is therefore global and matches the agent's uid in every distro (keep `agentUid` apart from - the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable. + the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable on + T3's port or `devPorts` (the agent's other localhost traffic is dropped). **Run no localhost servers in other + distros** of this PC on those ports. What the agent serves on those ports is forwarded to Windows' `localhost` by + WSL: open it in a browser only as you would an unknown site (a page on `localhost` shares cookies with every other + `localhost` page, whatever the port). Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. Another distro's root can drop or change the global nft table: the check sees it within 2 minutes and stops T3, and the unit's own `IPAddressDeny=` (per cgroup, not global) holds meanwhile. Abstract unix sockets are shared by every @@ -104,7 +115,20 @@ it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). change. An agent can still start its own programs (even its own copy of T3) inside the unit: those ignore the policy, but stay inside every other wall. - **Public internet is open** to the agent: the firewall blocks private ranges, not the public internet (GitHub's - and Anthropic's addresses are many and change). Add an outbound proxy if that matters. + and Anthropic's addresses are many and change). Add an outbound proxy if that matters. That includes the home's + own public address: a router that loops it back (hairpin NAT) lets the agent reach its admin page, if the router + serves it there, and every port-forwarded device of the home. Forward no ports to devices the agent must not + reach, and keep the router's admin page off the public side. +- **Servers in the agent's dev range only:** a test or dev server that listens on a port outside `devPorts` cannot + be reached over `localhost` (port 0, any free port, may be bound, but the agent cannot connect to it there). + Point them at `devPorts`, or widen it. +- **Shared with root's programs in this distro:** the system D-Bus (`/run/dbus`, its own access rules decide), + `/dev/shm` (mode 1777; others' files keep their modes), and `NETLINK_SOCK_DIAG`, which lists every distro's + sockets (addresses and ports, not their data). +- **DNS** goes to the public `nameservers` only. A `systemd-resolved` stub (`127.0.0.53`), which may resolve LAN + names through Windows, is out of reach: the agent's localhost traffic outside T3's port and `devPorts` is dropped. +- **Terminals:** what the agent prints in a terminal can use escape codes, such as OSC 52, which some terminals turn + into a write to the Windows clipboard. Do not paste from the clipboard blindly after watching agent output. - **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not run; the distro is the sandbox, so use full access. - **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py index 703fbab0a5062..930ddf4e4c1e6 100644 --- a/fork-kit/pc-agents/lib/pc_agents.py +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -12,6 +12,7 @@ raise SystemExit('pc-agents: PC_AGENTS_ROOT is for the tests only, never for root; unset it') OWNER = os.geteuid() if ROOT else 0 # who must own every file the kit writes HOME, USER = '/home/agent', 'agent' +NOLOGIN = '/usr/sbin/nologin' # the agent's login shell: no shell for `wsl -u agent` or su; T3 gets SHELL from its unit OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote (a path may have two mid-write) LOADED = '/run/pc-agents/firewall.loaded' # sha256 of the firewall file and `nft list` of it, saved when loaded @@ -78,9 +79,24 @@ def is_ip_list(value, net, glob=False, most=64): 'allow': (list, [], lambda v: is_ip_list(v, True)), 'peers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, True, most=16) if v else '1 to 16 entries'), 'tailscaleTag': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'tag:[a-z0-9-]{1,50}', v) else 'tag:name'), + 'devPorts': (str, '4100-4199', lambda v: None if port_range(v) else 'LOW-HIGH, 1024 to 65535, like 4100-4199'), } +def port_range(text): + """(low, high) of a 'LOW-HIGH' port range of 1024..65535, or None.""" + m = re.fullmatch(r'([1-9]\d{3,4})-([1-9]\d{3,4})', text) + low, high = (int(m[1]), int(m[2])) if m else (0, 0) + return (low, high) if 1024 <= low <= high <= 65535 else None + + +def loopback_ports(c): + """The TCP ports the agent may listen on and reach on localhost: T3's port and devPorts (nft and the unit's + SocketBindAllow=). Written apart unless the range holds T3's port (nft refuses overlapping set elements).""" + low, high = port_range(c['devPorts']) + return ([] if low <= c['port'] <= high else [str(c['port'])]) + [f'{low}-{high}'] + + def locked_problem(path, value): """Why T3 would refuse to lock path to value, where that is cheap to tell (T3 checks the rest and does not start on a policy it cannot apply: managedSettings.ts), or None. An object is a complete value: a whole setting @@ -130,7 +146,8 @@ def render(c): allow6 = [a for a in c['allow'] if ':' in a] nft = HEAD + 'table inet pc_agents\ndelete table inet pc_agents\ntable inet pc_agents {\n\tchain agent_out {\n' + \ '\t\ttype filter hook output priority filter; policy accept;\n' + \ - f'\t\tmeta skuid != {c["agentUid"]} accept\n\t\tct state established,related accept\n\t\toifname "lo" accept\n' + \ + f'\t\tmeta skuid != {c["agentUid"]} accept\n\t\tct state established,related accept\n' + \ + f'\t\toifname "lo" tcp dport {{ {", ".join(loopback_ports(c))} }} accept\n\t\toifname "lo" drop\n' + \ (f'\t\tip daddr {{ {", ".join(allow4)} }} accept\n' if allow4 else '') + \ (f'\t\tip6 daddr {{ {", ".join(allow6)} }} accept\n' if allow6 else '') + \ f'\t\tip daddr {{ {", ".join(BLOCKED4)} }} drop\n\t\tip6 daddr {{ {", ".join(BLOCKED6)} }} drop\n\t}}\n}}\n' @@ -146,7 +163,7 @@ def render(c): User={USER} Group={USER} Environment=HOME={HOME} T3CODE_HOME={HOME}/.t3 T3CODE_PORT={c["port"]} T3CODE_LAN=true -Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin SHELL=/bin/bash ExecStartPre=+{CHECK} load-firewall ExecStartPre=+{CHECK} check ExecStart={runtime(c)}/t3 serve @@ -166,6 +183,8 @@ def render(c): InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale IPAddressDeny={" ".join(BLOCKED4 + BLOCKED6)} IPAddressAllow={" ".join(c["peers"] + c["allow"])} +SocketBindDeny=any +{chr(10).join("SocketBindAllow=tcp:" + p for p in loopback_ports(c))} RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS SystemCallArchitectures=native @@ -193,8 +212,10 @@ def render(c): STATE + '/firewall.nft': (nft, 0o644), sd + UNIT: (unit, 0o644), # A check that does not succeed, even one that cannot start or crashes before it can stop T3, stops T3. - sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\n\n[Service]\nType=oneshot\n' - f'ExecStart={CHECK} check --enforce\n' + # After T3's unit: a check started while T3 is still starting (slow network at boot) waits for that start, + # instead of failing on a firewall not loaded yet and stopping T3 before it ran. + sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\nAfter={UNIT}\n\n[Service]\n' + f'Type=oneshot\nTimeoutStartSec=5min\nExecStart={CHECK} check --enforce\n' 'ExecStopPost=/bin/sh -c \'[ "$$SERVICE_RESULT" = success ] || ' f'exec /usr/bin/systemctl stop {UNIT}\'\n', 0o644), sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents walls check every 2 minutes\n\n[Timer]\n' @@ -289,6 +310,40 @@ def rows(path): {r[0]: [m for m in r[3].split(',') if m] for r in rows('/etc/group')}) +def login_shell(name): + """name's login shell in the passwd file, or None.""" + try: + with open(ROOT + '/etc/passwd', encoding='utf-8') as f: + return next((r[6] for r in (line.rstrip('\n').split(':') for line in f) if r[0] == name and len(r) > 6), + None) + except FileNotFoundError: + return None + + +def sudo_problem(c): + """Why sudo could give the agent more (a sudoers line naming it, its group or uid, or every user), or None. + Harmless inside the unit (NoNewPrivileges), but a way out from anywhere else.""" + who = re.compile(rf'(^|[\s,=:!(])(%?{USER}|%?#{c["agentUid"]}|ALL)($|[\s,=:)])') + found = [] + names = ['/etc/sudoers'] + (['/etc/sudoers.d/' + n for n in sorted(os.listdir(ROOT + '/etc/sudoers.d'))] + if os.path.isdir(ROOT + '/etc/sudoers.d') else []) + for path in names: + try: + with open(ROOT + path, encoding='utf-8', errors='replace') as f: + lines = f.read().replace('\\\n', ' ').splitlines() + except (FileNotFoundError, IsADirectoryError): + continue + for line in lines: + line = re.sub(r'#(?!include)(?!\d).*', '', line).strip() + if line.startswith('Defaults') or line.startswith(('@include', '#include')): + continue + first = line.split(None, 1)[0] if line else '' + # a user list (first field) or an alias naming the agent; `root ALL=(ALL) ALL` names root, not the agent + if line and (who.search(first) or re.match(r'User_Alias\b', line) and who.search(line.split('=', 1)[-1])): + found.append(f'{path}: {line[:80]}') + return found and 'sudo rules that cover the agent: ' + '; '.join(found[:3]) + + def user_problem(c, need_agent): """Why this distro is someone else's (another login user, an agent that is not the kit's), or None.""" people, groups = users() diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents index 73710c4c1ead4..188d680d13a44 100755 --- a/fork-kit/pc-agents/sbin/pc-agents +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -62,15 +62,22 @@ def own_walls(c, before_enable=False): seen = [] for u in k.UNITS: # what systemd has loaded, not only what is on disk r = k.run(['/usr/bin/systemctl', 'show', '-p', - 'LoadState,FragmentPath,DropInPaths,UnitFileState,NeedDaemonReload', u]) + 'LoadState,FragmentPath,DropInPaths,UnitFileState,NeedDaemonReload,ActiveState', u]) got = dict(line.split('=', 1) for line in r.stdout.splitlines() if '=' in line) want = {'LoadState': 'loaded', 'FragmentPath': f'{k.UNIT_DIRS[0]}/{u}', 'DropInPaths': '', 'NeedDaemonReload': 'no'} if u != k.WALLS + '.service' and not before_enable: want['UnitFileState'] = 'enabled' + if u == k.WALLS + '.timer' and not before_enable: # enabled is not enough: a stopped timer checks nothing + want['ActiveState'] = 'active' seen += [f'{u}: {key}={got.get(key)!r}' for key, v in want.items() if r.returncode or got.get(key) != v] - out.append(('units', 'systemd runs the kit\'s units as written, enabled', seen and '; '.join(seen[:3]))) - out.append(('agent', f'one login user, {k.USER}, in no group but its own', k.user_problem(c, True))) + out.append(('units', 'systemd runs the kit\'s units as written, enabled, the timer running', + seen and '; '.join(seen[:3]))) + shell = k.login_shell(k.USER) + out.append(('agent', f'one login user, {k.USER}, in no group but its own, with no login shell', + k.user_problem(c, True) or (None if shell == k.NOLOGIN else + f'{k.USER}\'s login shell is {shell}, not {k.NOLOGIN}'))) + out.append(('sudoers', f'no sudo rule covers {k.USER}', k.sudo_problem(c))) current = k.lstat(k.OPT + '/current') target = os.readlink(k.ROOT + k.OPT + '/current') if current and stat.S_ISLNK(current.st_mode) else '' code = [] if current and current.st_uid == k.OWNER and re.fullmatch(r'[0-9a-f]{40}', target) else \ diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py index 1f69c0b77b80b..de9717f11eec8 100644 --- a/fork-kit/pc-agents/tests/check.test.py +++ b/fork-kit/pc-agents/tests/check.test.py @@ -25,7 +25,7 @@ def test_all_hold(self): self.assertEqual(code, 0, out) walls = [line.split()[1] for line in out.splitlines()] for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', - 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'kit-code', 'runtime', + 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'sudoers', 'kit-code', 'runtime', 'automount', 'firewall', 'gateway', 'file:policy.json', 'file:t3code-agents.service', 'file:firewall.nft', 'file:wsl.conf', 'file:pc-agents-walls.timer'): self.assertIn(wall, walls) @@ -71,6 +71,7 @@ def edit(p, old, new): 'units': lambda: self.write('/log/units/t3code-agents.service.dropins', '/run/systemd/system/x.conf'), 'runtime': lambda: os.chmod(self.path(k.runtime(CONFIG) + '/t3'), 0o777), 'file:pc-agents-walls.timer': edit('/etc/systemd/system/pc-agents-walls.timer', '2min', '1d'), + 'sudoers': lambda: self.write('/etc/sudoers.d/90-agent', 'agent ALL=(ALL) NOPASSWD: ALL\n'), } clean = {p: v[4] for p, v in self.snapshot().items()} for wall, change in cases.items(): @@ -87,6 +88,28 @@ def again(self, **section): self.files() self.assertIsNone(k.load_firewall()) + def test_agent_has_no_login_shell(self): + self.write('/etc/passwd', self.read('/etc/passwd').replace('/home/agent:/usr/sbin/nologin', '/home/agent:/bin/bash')) + self.fails('agent') + + def test_sudo_rules_that_cover_the_agent(self): + base = 'Defaults env_reset\nroot ALL=(ALL:ALL) ALL\n%sudo ALL=(ALL:ALL) ALL\n@includedir /etc/sudoers.d\n' + self.write('/etc/sudoers', base + '# agent ALL=(ALL) ALL is only a comment\n') + self.write('/etc/sudoers.d/README', '# agent\n') + self.assertEqual(self.check()[0], 0) + for rule in ('agent ALL=(ALL) NOPASSWD: ALL', '%agent ALL=(ALL) ALL', '#1100 ALL=(ALL) ALL', 'ALL ALL=(ALL) ALL', + 'bob,agent ALL=(ALL) ALL', 'User_Alias AGENTS = bob, agent', 'agent \\\n ALL=(ALL) ALL'): + with self.subTest(rule=rule): + self.write('/etc/sudoers.d/x', rule + '\n') + self.fails('sudoers') + os.unlink(self.path('/etc/sudoers.d/x')) + self.write('/etc/sudoers', base + 'agent ALL=(ALL) ALL\n') + self.fails('sudoers') + + def test_timer_stopped_but_enabled(self): + os.unlink(self.path('/log/units/pc-agents-walls.timer.active')) + self.fails('units') + def test_agent_user_manager_masked(self): os.unlink(self.path('/etc/systemd/system/user@1100.service')) self.fails('masks') @@ -227,6 +250,46 @@ def test_unit(self): self.assertIn('\nStartLimitBurst=5\n', unit) self.assertIn('\n[user]\ndefault=root\n', files['/etc/wsl.conf'][0]) self.assertIn('\nOnUnitActiveSec=2min\n', files['/etc/systemd/system/pc-agents-walls.timer'][0]) + self.assertIn(' SHELL=/bin/bash\n', unit) # the agent's login shell is nologin; T3's shells are bash + walls = files['/etc/systemd/system/pc-agents-walls.service'][0] + for line in ('After=t3code-agents.service', 'TimeoutStartSec=5min', 'Type=oneshot'): + self.assertIn('\n' + line + '\n', walls) + + def test_loopback(self): + """The agent listens on and reaches over localhost only T3's port and devPorts: other distros' localhost + servers are out of reach, and what the agent serves is not forwarded to Windows' localhost on any port.""" + files = k.render(k.config()) + unit = files['/etc/systemd/system/t3code-agents.service'][0] + nft = files['/etc/pc-agents/firewall.nft'][0] + self.assertIn('\nSocketBindDeny=any\nSocketBindAllow=tcp:3800\nSocketBindAllow=tcp:4100-4199\n', unit) + self.assertIn('\t\toifname "lo" tcp dport { 3800, 4100-4199 } accept\n\t\toifname "lo" drop\n', nft) + self.assertNotIn('oifname "lo" accept', nft) + self.assertLess(nft.index('oifname "lo" drop'), nft.index('ip daddr {')) + self.machine({**CONFIG, 'devPorts': '3700-3899'}) # T3's port inside the range: written once (nft) + files = k.render(k.config()) + self.assertIn('\nSocketBindDeny=any\nSocketBindAllow=tcp:3700-3899\nRestrict', files['/etc/systemd/system/t3code-agents.service'][0]) + self.assertIn('tcp dport { 3700-3899 } accept', files['/etc/pc-agents/firewall.nft'][0]) + for bad in ('80-90', '5000-4000', '4100', '4100-70000', ' 4100-4199', 4100): + with self.subTest(devPorts=bad): + self.machine({**CONFIG, 'devPorts': bad}) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('devPorts', out) + + def test_loopback_rules_are_checked(self): + self.walled() + for name, change in (('unit', lambda: self.write('/etc/systemd/system/t3code-agents.service', self.read( + '/etc/systemd/system/t3code-agents.service').replace('SocketBindDeny=any\n', ''))), + ('loaded table', lambda: self.write('/log/nft-loaded', self.read('/log/nft-loaded').replace( + '\t\toifname "lo" drop\n', '')))): + with self.subTest(name): + shutil.rmtree(self.root) + self.setUp() + self.walled() + change() + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL ' + ('file:t3code-agents.service' if name == 'unit' else 'firewall'), out) def test_config(self): # The kit passes these; T3 itself tells a complete object from a partial one and does not start on a partial. diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py index 79f79362ac740..eab2f08c38cc6 100644 --- a/fork-kit/pc-agents/tests/fake_distro.py +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -13,8 +13,9 @@ 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'peers': ['100.101.102.103/32'], 'tailscaleTag': 'tag:pc-agents'} TOOLS = { # fake programs, under the fake root; each finds the root from its own path - 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; prev=; for a; do [ "$prev" = --uid ] && uid=$a; prev=$a; ' - 'done\necho "$a:x:$uid:$uid::/home/$a:/bin/bash" >>"$R/etc/passwd"\n' + 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; sh=/bin/sh; prev=; for a; do [ "$prev" = --uid ] && uid=$a; ' + '[ "$prev" = --shell ] && sh=$a; prev=$a; done\n' + 'echo "$a:x:$uid:$uid::/home/$a:$sh" >>"$R/etc/passwd"\n' 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', # T3's installer, run by root through env -i (scripts/install.sh with T3CODE_HOME=/opt/pc-agents/t3) 'usr/bin/env': 'R="${0%/usr/bin/env}"; echo "$*" >>"$R/log/install"\n[ -e "$R/log/install-fail" ] && exit 1\n' @@ -38,7 +39,8 @@ ' [ "$u" = pc-agents-walls.service ] && st=static\n' ' echo "LoadState=${f:+loaded}"; echo "FragmentPath=$f"\n' ' echo "DropInPaths=$(cat "$S/$u.dropins" 2>/dev/null)"; echo "UnitFileState=$st"\n' - ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no;;\n' + ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no\n' + ' [ -e "$S/$u.active" ] && echo ActiveState=active || echo ActiveState=inactive;;\n' 'esac\n', 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', @@ -160,7 +162,7 @@ def walled(self, sha=SHA1): os.makedirs(os.path.dirname(self.path(p)), exist_ok=True) os.symlink('/dev/null', self.path(p)) with open(self.path('/etc/passwd'), 'a') as f: - f.write('agent:x:1100:1100::/home/agent:/bin/bash\n') + f.write('agent:x:1100:1100::/home/agent:/usr/sbin/nologin\n') with open(self.path('/etc/group'), 'a') as f: f.write('agent:x:1100:\n') for src, (dst, mode) in k.CODE.items(): From 315aa3beecd0c3356e8f8507a9a8718e5d5636d1 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:06:20 +0000 Subject: [PATCH 4/8] fork-ci-tests: run test files in parallel so the step fits its 10-minute limit The serial run had grown to about 500 s of tests on a 4-core runner and was cancelled at the job limit. Test files now run FORK_CI_TEST_JOBS at once (default: the CPU count), each one's output printed whole in its own group. A process a test leaves behind holding its output no longer keeps the run waiting. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.cjs | 85 +++++++++++++++++++----- .github/scripts/fork-ci-tests.test.cjs | 90 +++++++++++++++++++++++++- 2 files changed, 159 insertions(+), 16 deletions(-) diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 9b4d31a5a5991..07ac124c3c848 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -2,8 +2,11 @@ // A plain entry runs here by its extension (RUNNER); ` run by ` is run by that file, // which must name the test and cannot be a (disabled) upstream workflow. Unlisted or runner-less // tests fail; a listed test that is gone fails once the tested history had it, a note before. +// Test files run in parallel, FORK_CI_TEST_JOBS at once (default: the CPU count); each one's output +// is printed whole, in its own group, when it ends. const childProcess = require("node:child_process"); const fs = require("node:fs"); +const os = require("node:os"); const path = require("node:path"); const ROOT = path.resolve(__dirname, "../.."); @@ -119,24 +122,72 @@ function readFile(file) { } } -function main() { +// Calls start(item) for every item, at most `jobs` at once, each as soon as a slot is free. +// start returns a promise; the results come back in the items' order. +async function runAll(items, jobs, start) { + const results = new Array(items.length); + let next = 0; + async function worker() { + while (next < items.length) { + const index = next++; + results[index] = await start(items[index]); + } + } + await Promise.all(Array.from({ length: Math.max(1, Math.min(jobs, items.length)) }, worker)); + return results; +} + +function jobCount(env = process.env) { + const jobs = Number(env.FORK_CI_TEST_JOBS); + return Number.isInteger(jobs) && jobs > 0 ? jobs : os.availableParallelism(); +} + +// Runs one test file, its stdout and stderr kept together, and prints them as one group when it ends. +function runTest(file) { + const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]]; + const started = Date.now(); + return new Promise((resolve) => { + const child = childProcess.spawn(command, [...args, file], { + cwd: ROOT, + stdio: ["ignore", "pipe", "pipe"], + }); + const output = []; + child.stdout.on("data", (chunk) => output.push(chunk)); + child.stderr.on("data", (chunk) => output.push(chunk)); + let ended = false; + const done = (status, signal) => { + if (ended) return; + ended = true; + // Stop reading pipes a left-behind process still holds, so they do not keep this script alive. + child.stdout.destroy(); + child.stderr.destroy(); + const seconds = ((Date.now() - started) / 1000).toFixed(1); + process.stdout.write(`::group::${file} (${seconds} s)\n`); + process.stdout.write(Buffer.concat(output)); + process.stdout.write("\n::endgroup::\n"); + if (status !== 0) + console.log(`::error file=${file}::${file} failed (exit ${status ?? signal})`); + resolve({ file, status, seconds }); + }; + child.on("error", (error) => { + output.push(Buffer.from(`cannot start ${command}: ${error.message}\n`)); + done(null, "spawn error"); + }); + child.on("close", done); + // A process the test left behind may hold the pipes open: the test is done when it exits. + child.on("exit", (status, signal) => setTimeout(() => done(status, signal), 5000).unref()); + }); +} + +async function main() { const entries = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8")); const result = plan(entries, presentTests(), everInHistory, readFile); const rows = []; let failed = result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length; - for (const file of result.run) { - const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]]; - const started = Date.now(); - console.log(`::group::${file}`); - const run = childProcess.spawnSync(command, [...args, file], { cwd: ROOT, stdio: "inherit" }); - console.log("::endgroup::"); - const seconds = ((Date.now() - started) / 1000).toFixed(1); - if (run.status !== 0) { - failed += 1; - console.log(`::error file=${file}::${file} failed (exit ${run.status ?? run.signal})`); - } - rows.push(`| \`${file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${seconds} s |`); + for (const run of await runAll(result.run, jobCount(), runTest)) { + if (run.status !== 0) failed += 1; + rows.push(`| \`${run.file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${run.seconds} s |`); } for (const file of result.unlisted) console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`); @@ -175,6 +226,10 @@ function main() { if (failed) process.exitCode = 1; } -module.exports = { plan, readList, findTests, presentTests }; +module.exports = { plan, readList, findTests, presentTests, runAll, jobCount }; -if (require.main === module) main(); +if (require.main === module) + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index da0290dc92daa..341da49ffe258 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -5,7 +5,7 @@ const os = require("node:os"); const path = require("node:path"); const test = require("node:test"); -const { plan, readList, findTests, presentTests } = require("./fork-ci-tests.cjs"); +const { plan, readList, findTests, presentTests, runAll, jobCount } = require("./fork-ci-tests.cjs"); const never = () => false; const here = (...files) => files.map((file) => ({ file, runBy: null })); @@ -143,3 +143,91 @@ test("an unlisted test under .github/scripts or fork-ci fails the run", () => { fs.rmSync(root, { recursive: true, force: true }); } }); + +test("runAll runs at most `jobs` at once, starts the next as soon as a slot frees, keeps the order", async () => { + let running = 0; + let most = 0; + const started = []; + const results = await runAll([30, 5, 5, 5, 5], 2, async (ms) => { + started.push(ms); + running += 1; + most = Math.max(most, running); + await new Promise((resolve) => setTimeout(resolve, ms)); + running -= 1; + return ms * 2; + }); + assert.equal(most, 2); + assert.deepEqual(results, [60, 10, 10, 10, 10]); + assert.equal(started.length, 5); + assert.deepEqual(await runAll([], 4, async () => 1), []); +}); + +test("jobCount reads FORK_CI_TEST_JOBS, else the CPU count", () => { + assert.equal(jobCount({ FORK_CI_TEST_JOBS: "3" }), 3); + for (const bad of [undefined, "", "0", "-2", "1.5", "many"]) + assert.equal(jobCount({ FORK_CI_TEST_JOBS: bad }), os.availableParallelism(), String(bad)); +}); + +// Runs the real script in a scratch tree whose list names `tests` ({ file: shell text }). +function runScript(tests, jobs) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fork-ci-tests-")); + const write = (file, text) => { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + }; + write(".github/scripts/fork-ci-tests.cjs", fs.readFileSync(require.resolve("./fork-ci-tests.cjs"))); + write(".github/fork-ci-tests.txt", Object.keys(tests).join("\n") + "\n"); + for (const [file, text] of Object.entries(tests)) write(file, text); + const env = { ...process.env, FORK_CI_TEST_JOBS: String(jobs), LEFT: path.join(root, "left.pid") }; + delete env.GITHUB_STEP_SUMMARY; + const started = Date.now(); + const run = childProcess.spawnSync("node", [".github/scripts/fork-ci-tests.cjs"], { + cwd: root, + env, + encoding: "utf8", + timeout: 60000, + }); + return { root, run, seconds: (Date.now() - started) / 1000 }; +} + +test("test files run in parallel, each one's output in its own group, and a failure fails the run", () => { + const { root, run, seconds } = runScript( + { + "fork-ci/a.test.sh": "echo a-start; sleep 2; echo a-end\n", + "fork-ci/b.test.sh": "echo b-start; sleep 2; echo b-end\n", + "fork-ci/c.test.sh": "echo c-start; sleep 2; echo c-end; exit 3\n", + }, + 3, + ); + try { + assert.equal(run.status, 1, run.stdout + run.stderr); + // One after another they take 6 s. + assert.ok(seconds < 5, `took ${seconds} s`); + for (const name of ["a", "b", "c"]) + assert.match( + run.stdout, + new RegExp(`::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`), + ); + assert.match(run.stdout, /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("a process a test leaves behind holding its output does not hang the run", () => { + const { root, run, seconds } = runScript( + { "fork-ci/left.test.sh": 'sleep 60 & echo $! > "$LEFT"; echo done\n' }, + 1, + ); + try { + assert.equal(run.error, undefined, String(run.error)); + assert.equal(run.status, 0, run.stdout + run.stderr); + assert.ok(seconds < 20, `took ${seconds} s`); + assert.match(run.stdout, /::group::fork-ci\/left\.test\.sh[^\n]*\ndone\n/); + } finally { + try { + process.kill(Number(fs.readFileSync(path.join(root, "left.pid"), "utf8"))); + } catch {} + fs.rmSync(root, { recursive: true, force: true }); + } +}); From 519449e8c462e7ebfd6c546c5b174f7f43e73dba Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:12:01 +0000 Subject: [PATCH 5/8] fork-ci-tests: format with vp fmt Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.cjs | 4 +++- .github/scripts/fork-ci-tests.test.cjs | 29 +++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 07ac124c3c848..2f42fe040fdf1 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -187,7 +187,9 @@ async function main() { result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length; for (const run of await runAll(result.run, jobCount(), runTest)) { if (run.status !== 0) failed += 1; - rows.push(`| \`${run.file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${run.seconds} s |`); + rows.push( + `| \`${run.file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${run.seconds} s |`, + ); } for (const file of result.unlisted) console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index 341da49ffe258..c064d1dd994fb 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -5,7 +5,14 @@ const os = require("node:os"); const path = require("node:path"); const test = require("node:test"); -const { plan, readList, findTests, presentTests, runAll, jobCount } = require("./fork-ci-tests.cjs"); +const { + plan, + readList, + findTests, + presentTests, + runAll, + jobCount, +} = require("./fork-ci-tests.cjs"); const never = () => false; const here = (...files) => files.map((file) => ({ file, runBy: null })); @@ -175,10 +182,17 @@ function runScript(tests, jobs) { fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); fs.writeFileSync(path.join(root, file), text); }; - write(".github/scripts/fork-ci-tests.cjs", fs.readFileSync(require.resolve("./fork-ci-tests.cjs"))); + write( + ".github/scripts/fork-ci-tests.cjs", + fs.readFileSync(require.resolve("./fork-ci-tests.cjs")), + ); write(".github/fork-ci-tests.txt", Object.keys(tests).join("\n") + "\n"); for (const [file, text] of Object.entries(tests)) write(file, text); - const env = { ...process.env, FORK_CI_TEST_JOBS: String(jobs), LEFT: path.join(root, "left.pid") }; + const env = { + ...process.env, + FORK_CI_TEST_JOBS: String(jobs), + LEFT: path.join(root, "left.pid"), + }; delete env.GITHUB_STEP_SUMMARY; const started = Date.now(); const run = childProcess.spawnSync("node", [".github/scripts/fork-ci-tests.cjs"], { @@ -206,9 +220,14 @@ test("test files run in parallel, each one's output in its own group, and a fail for (const name of ["a", "b", "c"]) assert.match( run.stdout, - new RegExp(`::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`), + new RegExp( + `::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`, + ), ); - assert.match(run.stdout, /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/); + assert.match( + run.stdout, + /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/, + ); } finally { fs.rmSync(root, { recursive: true, force: true }); } From f446ea1264c32294fe7096ca793f5eac651380af Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:30:52 +0000 Subject: [PATCH 6/8] fork-ci-tests: prove parallel runs without clocks; shorter header The runAll test drives controlled promises and checks start order and the most running at once; the script test has two files that each wait for the other's marker file, so they pass only when run side by side (and fail with FORK_CI_TEST_JOBS=1). No timers or Date.now in the tests. Header cut to 3 lines. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.cjs | 9 ++-- .github/scripts/fork-ci-tests.test.cjs | 68 +++++++++++++++++++------- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 2f42fe040fdf1..696b307ef5188 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -1,9 +1,6 @@ -// Fork only: every `*.test.*` under .github/scripts, fork-ci and fork-kit must be listed in .github/fork-ci-tests.txt. -// A plain entry runs here by its extension (RUNNER); ` run by ` is run by that file, -// which must name the test and cannot be a (disabled) upstream workflow. Unlisted or runner-less -// tests fail; a listed test that is gone fails once the tested history had it, a note before. -// Test files run in parallel, FORK_CI_TEST_JOBS at once (default: the CPU count); each one's output -// is printed whole, in its own group, when it ends. +// Fork only: every `*.test.*` under .github/scripts, fork-ci and fork-kit is listed in .github/fork-ci-tests.txt and +// runs here by extension (RUNNER), FORK_CI_TEST_JOBS at once (default: CPU count), or `run by ` (not an upstream +// workflow; it names the test). Unlisted or runner-less tests fail; a listed one that is gone fails once history had it. const childProcess = require("node:child_process"); const fs = require("node:fs"); const os = require("node:os"); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index c064d1dd994fb..1171f8153d5ac 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -152,20 +152,33 @@ test("an unlisted test under .github/scripts or fork-ci fails the run", () => { }); test("runAll runs at most `jobs` at once, starts the next as soon as a slot frees, keeps the order", async () => { + const runs = new Map(); // item -> resolve of its run, in start order + const waiting = []; // [count, resolve]: resolved once that many runs started + const started = (count) => + new Promise((resolve) => (runs.size >= count ? resolve() : waiting.push([count, resolve]))); let running = 0; let most = 0; - const started = []; - const results = await runAll([30, 5, 5, 5, 5], 2, async (ms) => { - started.push(ms); + const all = runAll(["a", "b", "c", "d"], 2, (item) => { running += 1; most = Math.max(most, running); - await new Promise((resolve) => setTimeout(resolve, ms)); - running -= 1; - return ms * 2; + const done = new Promise((resolve) => runs.set(item, resolve)).then(() => { + running -= 1; + return item.toUpperCase(); + }); + for (const [count, resolve] of waiting) if (runs.size >= count) resolve(); + return done; }); + await started(2); + assert.deepEqual([...runs.keys()], ["a", "b"]); // two at once, c waits for a free slot + runs.get("b")(); // the second ends first: its slot takes c + await started(3); + assert.deepEqual([...runs.keys()], ["a", "b", "c"]); + runs.get("a")(); + await started(4); + runs.get("d")(); + runs.get("c")(); + assert.deepEqual(await all, ["A", "B", "C", "D"]); assert.equal(most, 2); - assert.deepEqual(results, [60, 10, 10, 10, 10]); - assert.equal(started.length, 5); assert.deepEqual(await runAll([], 4, async () => 1), []); }); @@ -192,31 +205,35 @@ function runScript(tests, jobs) { ...process.env, FORK_CI_TEST_JOBS: String(jobs), LEFT: path.join(root, "left.pid"), + MARKS: root, }; delete env.GITHUB_STEP_SUMMARY; - const started = Date.now(); const run = childProcess.spawnSync("node", [".github/scripts/fork-ci-tests.cjs"], { cwd: root, env, encoding: "utf8", timeout: 60000, }); - return { root, run, seconds: (Date.now() - started) / 1000 }; + return { root, run }; } +// A test file that passes only when `other` starts while it runs (it waits up to 5 s for the other's mark). +const meets = (name, other) => + `echo ${name}-start; : > "$MARKS/${name}"\n` + + `for i in $(seq 50); do [ -e "$MARKS/${other}" ] && { echo ${name}-end; exit 0; }; sleep 0.1; done\n` + + `echo "${other} did not run beside ${name}"; exit 1\n`; + test("test files run in parallel, each one's output in its own group, and a failure fails the run", () => { - const { root, run, seconds } = runScript( + const { root, run } = runScript( { - "fork-ci/a.test.sh": "echo a-start; sleep 2; echo a-end\n", - "fork-ci/b.test.sh": "echo b-start; sleep 2; echo b-end\n", - "fork-ci/c.test.sh": "echo c-start; sleep 2; echo c-end; exit 3\n", + "fork-ci/a.test.sh": meets("a", "b"), + "fork-ci/b.test.sh": meets("b", "a"), + "fork-ci/c.test.sh": "echo c-start; echo c-end; exit 3\n", }, 3, ); try { assert.equal(run.status, 1, run.stdout + run.stderr); - // One after another they take 6 s. - assert.ok(seconds < 5, `took ${seconds} s`); for (const name of ["a", "b", "c"]) assert.match( run.stdout, @@ -224,6 +241,7 @@ test("test files run in parallel, each one's output in its own group, and a fail `::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`, ), ); + assert.doesNotMatch(run.stdout, /::error file=fork-ci\/[ab]\.test\.sh/); assert.match( run.stdout, /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/, @@ -233,15 +251,27 @@ test("test files run in parallel, each one's output in its own group, and a fail } }); +test("FORK_CI_TEST_JOBS=1 runs one file at a time", () => { + const { root, run } = runScript( + { "fork-ci/a.test.sh": meets("a", "b"), "fork-ci/b.test.sh": meets("b", "a") }, + 1, + ); + try { + assert.equal(run.status, 1, run.stdout + run.stderr); + assert.match(run.stdout, /b did not run beside a/); // b started only after a ended + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + test("a process a test leaves behind holding its output does not hang the run", () => { - const { root, run, seconds } = runScript( + const { root, run } = runScript( { "fork-ci/left.test.sh": 'sleep 60 & echo $! > "$LEFT"; echo done\n' }, 1, ); try { - assert.equal(run.error, undefined, String(run.error)); + assert.equal(run.error, undefined, String(run.error)); // not killed at the 60 s limit: the run ended assert.equal(run.status, 0, run.stdout + run.stderr); - assert.ok(seconds < 20, `took ${seconds} s`); assert.match(run.stdout, /::group::fork-ci\/left\.test\.sh[^\n]*\ndone\n/); } finally { try { From d6f9e208521046a4dd415a8ac6921490db125a41 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:45:07 +0000 Subject: [PATCH 7/8] fork tests: no pipe into a reader that stops early (SIGPIPE race) `sed ... | grep -q` under pipefail fails when grep exits at the match while sed still writes: a race (5 in 2000 runs idle, 45 under load) that the parallel runner made show up in fork-follow.test.sh. Ten such pipes (grep -q, head) now read the whole output first; two of them failed open (`! ... | grep -q`). fork-ci-tests.test.cjs fails on any new one in a listed shell test. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.test.cjs | 41 +++++++++++++++++++ .github/scripts/fork-follow.test.sh | 14 +++---- .../scripts/fork-interaction-limit.test.sh | 4 +- .github/scripts/fork-secret-scan.test.sh | 2 +- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index 1171f8153d5ac..3fbda8b020161 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -280,3 +280,44 @@ test("a process a test leaves behind holding its output does not hang the run", fs.rmSync(root, { recursive: true, force: true }); } }); + +// A pipe into a reader that stops early (`grep -q`, `grep -m`, `head`) kills the writer with SIGPIPE +// whenever it has more to write; under `pipefail` that reads as a failed check, more often under load. +const earlyExitPipe = + /(? + text + .split("\n") + .map((line, index) => ({ line, number: index + 1 })) + .filter(({ line }) => !/^\s*#/.test(line) && earlyExitPipe.test(line)); + +test("a pipe into grep -q, grep -m or head is found; reading the whole output first is not", () => { + const found = earlyExitPipes( + [ + 'sed -n "/a/,$p" f | grep -q x', + "! calls | grep -vqE y", + "git show c | grep -qx z", + "seq 9 | grep -m1 3", + 'sed p <<< "$s" | head -n 1', + 'grep -q x <<< "$(sed -n p f)"', + "a || grep -q x f", + "# a comment: `x | grep -q` stops early", + '[[ -z "$(grep -E a f | grep -vE b)" ]]', + ].join("\n"), + ).map(({ number }) => number); + assert.deepEqual(found, [1, 2, 3, 4, 5]); +}); + +test("no listed shell test pipes into a reader that stops early", () => { + const root = path.resolve(__dirname, "../.."); + const shellTests = readList(fs.readFileSync(path.join(root, ".github/fork-ci-tests.txt"), "utf8")) + .map(({ file }) => file) + .filter((file) => file.endsWith(".sh") && fs.existsSync(path.join(root, file))); + assert.ok(shellTests.length > 0); + const found = shellTests.flatMap((file) => + earlyExitPipes(fs.readFileSync(path.join(root, file), "utf8")).map( + ({ line, number }) => `${file}:${number}: ${line.trim()}`, + ), + ); + assert.deepEqual(found, []); +}); diff --git a/.github/scripts/fork-follow.test.sh b/.github/scripts/fork-follow.test.sh index 72eafc76779d3..cb5a252047a79 100755 --- a/.github/scripts/fork-follow.test.sh +++ b/.github/scripts/fork-follow.test.sh @@ -441,7 +441,7 @@ plan check "plan: upstream edits a deleted listed file -> the conflict is resolved by deleting it, rebase" \ eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.11 ]] && ! in_candidate ci.yml && [[ ! -e "$root/report.md" ]]' check "plan: the resolved rebase keeps the fork's patches and its list" \ - eval 'action_is rebase && git -C "$root/work" cat-file -e "$(out candidate):feature.txt" && git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt" | grep -qx .github/workflows/ci.yml' + eval 'action_is rebase && git -C "$root/work" cat-file -e "$(out candidate):feature.txt" && grep -qx .github/workflows/ci.yml <<< "$(git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt")"' # Re-run: an earlier run published this candidate, then stopped before the branch moved. The # next run rebuilds the same files and advances to that release instead of building again. released="$(out candidate)" @@ -470,7 +470,7 @@ new_fork v0.1.11 fork_commit "fork: list ci.yml" 'echo .github/workflows/ci.yml >> .github/fork-removed-workflows.txt' plan check "plan: upstream brings a listed file back -> deleted again on the candidate, rebase" \ - eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.12 ]] && ! in_candidate ci.yml && git -C "$root/work" log -1 --format=%s "$(out candidate)" | grep -q "fork: delete upstream workflows listed"' + eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.12 ]] && ! in_candidate ci.yml && grep -q "fork: delete upstream workflows listed" <<< "$(git -C "$root/work" log -1 --format=%s "$(out candidate)")"' new_upstream new_fork v0.1.10 @@ -506,7 +506,7 @@ new_fork v0.1.10 fork_commit "fork: delete ci.yml, CRLF list" "printf '# deleted\\r\\n.github/workflows/ci.yml\\r\\n' > .github/fork-removed-workflows.txt && rm .github/workflows/ci.yml" plan check "plan: a list with CRLF line ends is read like one with LF, the listed file stays deleted" \ - eval 'action_is rebase && ! in_candidate ci.yml && git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt" | grep -q $'"'"'ci.yml\r$'"'"'' + eval 'action_is rebase && ! in_candidate ci.yml && grep -q $'"'"'ci.yml\r$'"'"' <<< "$(git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt")"' # Upstream renames a listed file: git reports a rename/delete conflict on the new name, which the # list does not name, so the plan stops and names it rather than shipping it under that name. @@ -1054,9 +1054,9 @@ check "workflow: advance's if: is a conjunction (no top-level ||) of the plan's check "workflow: the move step runs only when the checks before it passed (no if:)" \ eval 'awk "/- name: Move the series branch/ { s = 1; next } s && /^ - / { exit } s && /^ if:/ { bad = 1 } END { exit bad }" <<< "$advance_job"' check "workflow: action none deletes a leftover candidate branch" \ - eval 'grep -B2 -A3 "action == .none." "$workflow" | grep -q "fork-follow-push.sh cleanup"' + eval 'grep -q "fork-follow-push.sh cleanup" <<< "$(grep -B2 -A3 "action == .none." "$workflow")"' check "workflow: the last sweep reports runs since the run began" \ - eval 'sed -n "/^ report:/,\$p" "$workflow" | grep -q "SINCE: \${{ needs.guard.outputs.started }}"' + eval 'grep -q "SINCE: \${{ needs.guard.outputs.started }}" <<< "$(sed -n "/^ report:/,\$p" "$workflow")"' # Each job that reads FORK_PUSH_TOKEN runs in the Environment that holds it, named as in env. token_jobs="$(grep -vF "$presence" "$workflow" | awk '/^ [a-z_-]+:$/ { job = $1 } /secrets\.FORK_PUSH_TOKEN/ { print job }' | sort -u)" env_jobs="$(awk '/^ [a-z_-]+:$/ { job = $1 } /^ environment: fork-push$/ { print job }' "$workflow" | sort -u)" @@ -1067,9 +1067,9 @@ check "workflow: the jobs that read FORK_PUSH_TOKEN, and only they, run in the E check "workflow: no job uses actions/checkout; four plain-git checkouts, none with a token" \ eval '! grep -q "actions/checkout@" "$workflow" && [[ $(grep -c "^ git init -q$" "$workflow") == 4 ]]' check "workflow: every action is pinned by commit" \ - eval '! grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vqE "@[0-9a-f]{40}( |$)"' + eval '[[ -z "$(grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vE "@[0-9a-f]{40}( |$)")" ]]' check "workflow: the report runs on every run (also action none)" \ - eval 'sed -n "/^ report:/,\$p" "$workflow" | grep -q "^ if: always()$"' + eval 'grep -q "^ if: always()$" <<< "$(sed -n "/^ report:/,\$p" "$workflow")"' check "workflow: upstream workflows are swept after each push" \ eval '[[ $(grep -c "fork-workflows.sh" "$workflow") -ge 3 ]]' diff --git a/.github/scripts/fork-interaction-limit.test.sh b/.github/scripts/fork-interaction-limit.test.sh index 63424f684b844..0c6a499be68c3 100644 --- a/.github/scripts/fork-interaction-limit.test.sh +++ b/.github/scripts/fork-interaction-limit.test.sh @@ -107,8 +107,8 @@ expect() { # expect LABEL 'CONDITION' (shell, evaluated) } rc_is() { [[ "$rc" == "$1" ]]; } says() { grep -qF -- "$1" <<< "$out"; } -called() { calls | grep -qF -- "$1"; } -not_called() { ! calls | grep -qF -- "$1"; } +called() { grep -qF -- "$1" <<< "$(calls)"; } +not_called() { ! grep -qF -- "$1" <<< "$(calls)"; } # --- check ------------------------------------------------------------------------------------- reset; limit collaborators_only 180; run check diff --git a/.github/scripts/fork-secret-scan.test.sh b/.github/scripts/fork-secret-scan.test.sh index fdddfabb60641..675a9713b543f 100755 --- a/.github/scripts/fork-secret-scan.test.sh +++ b/.github/scripts/fork-secret-scan.test.sh @@ -61,7 +61,7 @@ elif [[ -n "${GITLEAKS_CONFIG:-}" ]]; then settings="$(cat "$GITLEAKS_CONFIG")" elif [[ -n "${GITLEAKS_CONFIG_TOML:-}" ]]; then settings="$GITLEAKS_CONFIG_TOML" elif [[ -f "$source/.gitleaks.toml" ]]; then settings="$(cat "$source/.gitleaks.toml")" fi -extend_path="$(sed -nE "s/^[[:space:]]*path[[:space:]]*=[[:space:]]*[\"']([^\"']*)[\"'].*/\1/p" <<< "$settings" | head -n 1)" +extend_path="$(sed -nE "s/^[[:space:]]*path[[:space:]]*=[[:space:]]*[\"']([^\"']*)[\"'].*/\1/p" <<< "$settings" | sed -n 1p)" if [[ "$settings" == *"[extend]"* && -n "$extend_path" && ! -f "$extend_path" ]]; then echo "FTL Failed to load config error=\"failed to load extended config, err: open $extend_path: no such file or directory\"" exit 1 From d9ce01555815a02ed77f3dbc0e090c726f21b699 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:51:45 +0000 Subject: [PATCH 8/8] fork-run-alert test: read calls fully before grep -q Main's fork-run-alert.test.sh had the same early-exit pipe the new guard refuses; read the whole output first, as in the other shell tests. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-run-alert.test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/fork-run-alert.test.sh b/.github/scripts/fork-run-alert.test.sh index ea5e77c6751dc..ca4311e340760 100644 --- a/.github/scripts/fork-run-alert.test.sh +++ b/.github/scripts/fork-run-alert.test.sh @@ -83,8 +83,8 @@ expect() { # expect LABEL 'CONDITION' (shell, evaluated) } rc_is() { [[ "$rc" == "$1" ]]; } says() { grep -qF -- "$1" <<< "$out"; } -called() { calls | grep -qF -- "$1"; } -not_called() { ! calls | grep -qF -- "$1"; } +called() { grep -qF -- "$1" <<< "$(calls)"; } +not_called() { ! grep -qF -- "$1" <<< "$(calls)"; } issues() { jq length "$root/state/issues.json" 2> /dev/null || echo 0; } reset; run failure --message "The daily secret sweep failed"