From 69a273eb764f5830f8ba7799c8042a67301b0ad1 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 09:59:51 +0000 Subject: [PATCH 1/2] fork fleet: desk link, always-on side (D2) serve: the ssh forced command a desktop's agent dials into; one session per pool (lock), a private socket, pings, and "outcome unknown" for every request in flight when the session ends. launch: a pool's launch command that sends one launcher verb through the live session; no session = exit 3 at once, no answer in time = exit 75. The wire format both sides load lives in fork-fleet-desk-wire.py, with a contract test against the launcher and the dispatcher. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 1 + .github/fork-fleet.md | 21 ++ .github/scripts/fork-fleet-desk-wire.py | 178 +++++++++++++ .github/scripts/fork-fleet-desk.py | 332 ++++++++++++++++++++++++ .github/scripts/fork-fleet-desk.test.py | 298 +++++++++++++++++++++ 5 files changed, 830 insertions(+) create mode 100644 .github/scripts/fork-fleet-desk-wire.py create mode 100755 .github/scripts/fork-fleet-desk.py create mode 100644 .github/scripts/fork-fleet-desk.test.py diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 3c72443109c9f..d13abae8b11c0 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -7,6 +7,7 @@ # is a note; one that was present and then deleted fails. .github/scripts/fork-ci-tests.test.cjs .github/scripts/fork-contracts.test.py +.github/scripts/fork-fleet-desk.test.py .github/scripts/fork-fleet.test.py .github/scripts/fork-gate-trust-docs.test.py .github/scripts/fork-head-gate-workflow.test.py diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index 50a6dfcefd66f..69824fc5e59aa 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -464,6 +464,27 @@ that runs `fork-fleet-launch.sh` on the PC over ssh. Run the dispatcher with `--pool vm` until then. Without its dispatcher, `pc` has no heartbeat, and `route` never counts it. Whether and how the PC joins is an owner decision. +### The desk link (a desktop pool) + +A desktop accepts no connection and holds no GitHub token. Its desk agent dials +out to the always-on host over ssh, with a key that may only run `serve` for +its own pool, and the pool's dispatcher on the always-on host sends the +launcher's verbs through that one session (`fork-fleet-desk.py`, its wire +format `fork-fleet-desk-wire.py`): + +``` +# root's authorized_keys on the always-on host (sshd: PermitRootLogin forced-commands-only), one line per pool: +restrict,command="python3 -I /opt/fork-fleet/fork-fleet-desk.py serve POOL" ssh-ed25519 AAAA... desk-POOL +# the pool's launch command in the project's fleet config: +"launch": ["python3", "-I", "/opt/fork-fleet/fork-fleet-desk.py", "launch", "POOL"] +``` + +`serve` keeps one session per pool (a lock), listens on a root-only socket in +`/run/fork-fleet-desk`, and pings the agent; `launch` fails at once (exit 3) +when no session is live, so a sleeping desktop's pool is down for the pass and +`route` sends its jobs elsewhere. A verb with no answer in its time exits 75: +it may or may not have run, and the next pass's `active` tells. + **What jobs need on the host.** The jobs run without sudo, so the host must already have: diff --git a/.github/scripts/fork-fleet-desk-wire.py b/.github/scripts/fork-fleet-desk-wire.py new file mode 100644 index 0000000000000..5c4fc4ffffd67 --- /dev/null +++ b/.github/scripts/fork-fleet-desk-wire.py @@ -0,0 +1,178 @@ +"""Fork only: the wire format of the desk link (.github/fork-fleet.md, "The desk link"), the one module both of its +sides load: `fork-fleet-desk.py` on the always-on host (`serve`, `launch`) and the desk agent on the desktop. + +The link: the desktop dials out to the always-on host over ssh with a key that may only run +`fork-fleet-desk.py serve POOL`. Through that session the always-on host's `launch POOL VERB ARGS...` (a pool's +`launch` command for `fork-fleet.py dispatch`) has the desktop run `fork-fleet-launch.sh VERB ARGS...`. + +Frames: one JSON object per line (ASCII, "\n"-terminated, at most MAX_FRAME bytes), each with "v": VERSION. + agent -> serve, first {"v":1, "hello": POOL} the pool the agent serves (must be serve's) + serve -> agent, answer {"v":1, "hello": POOL} + serve -> agent {"v":1, "id": N, "verb": V, "args": [...], "stdin": S} one launcher call ("stdin": start only) + agent -> serve {"v":1, "id": N, "exit": E, "stdout": O, "stderr": R} its result; any order, once per id + serve -> agent {"v":1, "ping": N} the agent answers {"v":1, "pong": N} at once, also while verbs run + launch -> serve (socket) a request without "id"; serve answers with an answer without "id". +Anything else ends the session (or drops that one socket client). Standard library only. +""" +import json, re + +VERSION = 1 +MAX_FRAME = 256 * 1024 +MAX_TEXT = 64 * 1024 # stdout or stderr of one answer +POOL = re.compile(r'[a-z0-9][a-z0-9_-]{0,39}') # fork-fleet.py's POOL +NAME = re.compile(r'[a-z0-9][a-z0-9-]{0,62}-[0-9]{1,20}') # fork-fleet-launch.sh's NAME +RUNNER_VERSION = re.compile(r'[0-9]{1,5}\.[0-9]{1,5}\.[0-9]{1,5}') # fork-fleet-launch.sh's VERSION +MB = re.compile(r'[0-9]{1,7}') +URL = re.compile(r'https?://[A-Za-z0-9.-]+(:[0-9]{1,5})?(/[A-Za-z0-9._~/-]*)?') +HOST_PORT = re.compile(r'[0-9]{1,3}(\.[0-9]{1,3}){3}:[0-9]{1,5}') +START_LINE = re.compile(r'(ACTIONS_RUNNER_INPUT_JITCONFIG=[A-Za-z0-9+/=]{1,65536}|FORK_FLEET_(RUN|ATTEMPT)=[1-9][0-9]{0,19})') +# The launcher's verbs: (argument patterns, then a pattern every further argument matches or None, seconds the desktop +# may take). fork-fleet.py's Host waits 120 s for its launch command: every verb it calls answers before that. +VERBS = { + 'start': ((NAME,), None, 100), + 'ready': ((RUNNER_VERSION,), None, 30), + 'active': ((), None, 30), + 'fits': ((MB,), None, 30), + 'stop': ((NAME,), None, 100), + 'prune': ((), None, 100), + 'check-net': ((URL, HOST_PORT), HOST_PORT, 300), +} +MARGIN = 10 # seconds `launch` waits beyond a verb's own, for the link itself +UNKNOWN = 75 # launch's exit when the verb may or may not have run (EX_TEMPFAIL) +NO_SESSION = 3 # launch's exit when no desk session is live: nothing was sent +CONTROL = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\x9f]') + + +class WireError(ValueError): + pass + + +def encode(frame): + data = json.dumps(frame, separators=(',', ':'), ensure_ascii=True).encode() + b'\n' + if len(data) > MAX_FRAME: + raise WireError('frame too long') + return data + + +class Reader: + """Bytes in, frames (dicts) out. WireError on a line that is too long or not a JSON object of this VERSION.""" + + def __init__(self): + self.buf = b'' + + def feed(self, data): + self.buf += data + frames = [] + while b'\n' in self.buf: + line, self.buf = self.buf.split(b'\n', 1) + frames.append(decode(line)) + if len(self.buf) > MAX_FRAME: + raise WireError('frame too long') + return frames + + +def decode(line): + if len(line) + 1 > MAX_FRAME: + raise WireError('frame too long') + try: + frame = json.loads(line.decode('ascii')) + except (UnicodeDecodeError, ValueError): + raise WireError('not a JSON line') from None + if not isinstance(frame, dict) or frame.get('v') != VERSION or type(frame['v']) is not int: + raise WireError(f'not a version {VERSION} frame') + return frame + + +def _exact(frame, keys): + if set(frame) != {'v', *keys}: + raise WireError(f'a frame with keys {sorted(frame)}') + + +def _count(value): + if type(value) is not int or not 0 <= value < 2 ** 53: + raise WireError('not a frame number') + return value + + +def check_hello(frame, pool): + _exact(frame, ('hello',)) + if frame['hello'] != pool: + raise WireError(f'hello for pool {str(frame["hello"])[:40]!r}, not {pool}') + return frame + + +def check_ping(frame, key): + _exact(frame, (key,)) + return _count(frame[key]) + + +def check_start_input(text): + """The launcher's `start` input: its three lines, each once, nothing else.""" + lines = text.split('\n') + if lines and lines[-1] == '': + lines.pop() + keys = [line.split('=', 1)[0] for line in lines] + if sorted(keys) != ['ACTIONS_RUNNER_INPUT_JITCONFIG', 'FORK_FLEET_ATTEMPT', 'FORK_FLEET_RUN'] or \ + not all(START_LINE.fullmatch(line) for line in lines): + raise WireError('start: the input is not the three launcher lines') + + +def request(verb, args, stdin=None, ident=None): + """A checked request frame (`ident` None: the socket's form, without "id").""" + frame = {'v': VERSION, 'verb': verb, 'args': list(args)} + if stdin is not None: + frame['stdin'] = stdin + if ident is not None: + frame['id'] = ident + return check_request(frame, with_id=ident is not None) + + +def check_request(frame, with_id): + keys = {'verb', 'args'} | ({'id'} if with_id else set()) | ({'stdin'} if frame.get('verb') == 'start' else set()) + _exact(frame, keys) + if with_id: + _count(frame['id']) + verb, args = frame['verb'], frame['args'] + if verb not in VERBS or not isinstance(args, list) or not all(isinstance(a, str) for a in args): + raise WireError(f'not a launcher verb: {str(verb)[:40]!r}') + fixed, rest, _ = VERBS[verb] + if len(args) < len(fixed) or (rest is None and len(args) > len(fixed)): + raise WireError(f'{verb}: {len(args)} arguments') + for i, arg in enumerate(args): + if not (fixed[i] if i < len(fixed) else rest).fullmatch(arg): + raise WireError(f'{verb}: argument {i + 1} is not well formed') + if verb == 'start': + if not isinstance(frame['stdin'], str): + raise WireError('start: no input') + check_start_input(frame['stdin']) + return frame + + +def seconds(verb): + """How long `launch` waits for an answer to `verb`.""" + return VERBS[verb][2] + MARGIN + + +def clean(text): + """Text shown to a person or a log: no control characters but newline and tab.""" + return CONTROL.sub('?', text) + + +def answer(exit_code, stdout='', stderr='', ident=None): + frame = {'v': VERSION, 'exit': exit_code, 'stdout': stdout, 'stderr': stderr} + if ident is not None: + frame['id'] = ident + return check_answer(frame, with_id=ident is not None) + + +def check_answer(frame, with_id): + _exact(frame, ('exit', 'stdout', 'stderr') + (('id',) if with_id else ())) + if with_id: + _count(frame['id']) + if type(frame['exit']) is not int or not 0 <= frame['exit'] <= 255: + raise WireError('answer: exit is not 0 to 255') + for key in ('stdout', 'stderr'): + if not isinstance(frame[key], str) or len(frame[key]) > MAX_TEXT: + raise WireError(f'answer: {key} is not text of at most {MAX_TEXT} characters') + frame[key] = clean(frame[key]) + return frame diff --git a/.github/scripts/fork-fleet-desk.py b/.github/scripts/fork-fleet-desk.py new file mode 100755 index 0000000000000..325b17237f3fb --- /dev/null +++ b/.github/scripts/fork-fleet-desk.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +"""Fork only: the always-on host's side of the desk link (.github/fork-fleet.md, "The desk link"), as root. + +A desktop accepts no connection and holds no GitHub token: its desk agent dials out to the always-on host over ssh, +with a key that may only run `serve` for its own pool, and the pool's runners are started through that session. + + python3 -I fork-fleet-desk.py serve POOL + The ssh forced command (authorized_keys: `restrict,command="python3 -I /opt/fork-fleet/fork-fleet-desk.py serve + POOL" ssh-ed25519 ...`). Takes DIR/POOL.lock (one session per pool: a second one exits 3 at once), reads the + agent's hello, then listens on the socket DIR/POOL.sock (0600 in a 0700 DIR, both this user's) and passes each + request to the agent and its answer back. Pings the agent every PING s; with no word from it in DEAD s, or on + its end of the session, every request in flight is answered "outcome unknown" and the socket is removed. + python3 -I fork-fleet-desk.py launch POOL VERB ARGS... + A pool's `launch` command for `fork-fleet.py dispatch`: ["python3", "-I", ".../fork-fleet-desk.py", "launch", + POOL]. Sends one launcher verb (`start` with its stdin) through the live session and prints the desktop's + stdout and stderr and exits with its exit code. No live session: exit 3 at once, nothing sent. No answer in + the verb's time (fork-fleet-desk-wire.py VERBS): exit 75, the verb may or may not have run. +DIR is /run/fork-fleet-desk, root's. Not as root (tests), $FORK_FLEET_DESK_DIR may name another one. The link keeps no +state of its own: the live session is the state. Wire format: fork-fleet-desk-wire.py. Standard library only. +""" +import importlib.util, os, selectors, signal, socket, stat, struct, sys, time + +sys.dont_write_bytecode = True # a root-owned install directory gets no __pycache__ +HERE = os.path.dirname(os.path.realpath(__file__)) +_spec = importlib.util.spec_from_file_location('fork_fleet_desk_wire', os.path.join(HERE, 'fork-fleet-desk-wire.py')) +wire = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(wire) + +DIR = '/run/fork-fleet-desk' +HELLO = 30 # seconds the agent has for its hello +PING, DEAD = 15, 45 +CLIENT = 5 # seconds a socket client has to send its request, and to take its answer + + +class Refused(Exception): + def __init__(self, msg, code=1): + super().__init__(msg) + self.code = code + + +def sock_dir(env=os.environ, euid=None): + euid = os.geteuid() if euid is None else euid + return DIR if euid == 0 else env.get('FORK_FLEET_DESK_DIR') or DIR + + +def safe_dir(path, owners, make=False): + """The socket directory: not a symlink, owned by one of `owners`, no access for anyone else.""" + if make: + try: + os.mkdir(path, 0o700) + except FileExistsError: + pass + st = os.lstat(path) + if not stat.S_ISDIR(st.st_mode) or st.st_uid not in owners or st.st_mode & 0o077: + raise Refused(f'{path} must be a directory of uid {owners[0]}, mode 0700') + return st.st_uid + + +def pool_ok(pool): + if not wire.POOL.fullmatch(pool): + raise Refused(f'not a pool name: {pool[:40]!r}', 2) + + +# ---------------------------------------------------------------- serve + +def serve(pool, path, inp=0, out=1, log=sys.stderr, ping=PING, dead=DEAD, hello_wait=HELLO, now=time.monotonic): + pool_ok(pool) + safe_dir(path, (os.geteuid(),), make=True) + lock = os.open(os.path.join(path, f'{pool}.lock'), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + try: + import fcntl + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + raise Refused(f'another desk session serves pool {pool}', wire.NO_SESSION) from None + return Session(pool, path, inp, out, log, ping, dead, hello_wait, now).run() + finally: + os.close(lock) + + +class Session: + def __init__(self, pool, path, inp, out, log, ping, dead, hello_wait, now): + self.pool, self.inp, self.out, self.log, self.now = pool, inp, out, log, now + self.ping, self.dead, self.hello_wait = ping, dead, hello_wait + self.sock_path = os.path.join(path, f'{pool}.sock') + self.reader, self.outbuf, self.sel = wire.Reader(), bytearray(), selectors.DefaultSelector() + self.pending = {} # request id: (client socket, verb, deadline) + self.waiting = {} # client socket: (bytes so far, deadline) + self.next_id, self.pings = 1, 0 + + def say(self, msg): + print(f'fork-fleet-desk serve {self.pool}: {msg}', file=self.log, flush=True) + + def run(self): + os.set_blocking(self.out, False) + self.sel.register(self.inp, selectors.EVENT_READ, 'agent') + listener = None + try: + hello = self.agent_frames(self.now() + self.hello_wait) + wire.check_hello(hello, self.pool) + self.send(hello) + try: + os.unlink(self.sock_path) # a socket left by a session that died: this one holds the lock + except FileNotFoundError: + pass + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + old = os.umask(0o177) + try: + listener.bind(self.sock_path) + finally: + os.umask(old) + listener.listen(8) + listener.setblocking(False) + self.sel.register(listener, selectors.EVENT_READ, 'listen') + self.say('session open') + return self.loop(listener) + except (wire.WireError, Refused, OSError) as e: + self.say(f'session ends: {e}') + return 1 + finally: + if listener is not None: + try: + os.unlink(self.sock_path) + except FileNotFoundError: + pass + listener.close() + for ident in list(self.pending): + self.reply(ident, wire.answer(wire.UNKNOWN, stderr=self.unknown(self.pending[ident][1]))) + for client in list(self.waiting): + self.drop(client) + + def unknown(self, verb): + return f'fork-fleet-desk: pool {self.pool}: the desk session ended: {verb} may or may not have run\n' + + def agent_frames(self, deadline): + """The first frame from the agent (the hello), waiting until `deadline`.""" + while self.now() < deadline: + for key, _ in self.sel.select(max(0.0, deadline - self.now())): + frames = self.read_agent() + if len(frames) > 1: + raise wire.WireError('frames before the hello was answered') + if frames: + return frames[0] + raise Refused(f'no hello in {self.hello_wait} s') + + def read_agent(self): + try: + data = os.read(self.inp, 65536) + except BlockingIOError: + return [] + if not data: + raise Refused('the desktop ended the session') + self.heard, self.pings = self.now(), 0 + return self.reader.feed(data) + + def send(self, frame): + self.outbuf += wire.encode(frame) + self.flush() + + def flush(self): + try: + while self.outbuf: + self.outbuf = self.outbuf[os.write(self.out, self.outbuf):] + except BlockingIOError: + pass + + def loop(self, listener): + self.heard = self.now() + while True: + for key, _ in self.sel.select(min(1.0, self.ping / 4) if not self.outbuf else 0.05): + if key.data == 'agent': + for frame in self.read_agent(): + self.from_agent(frame) + elif key.data == 'listen': + self.accept(listener) + else: + self.from_client(key.fileobj) + self.flush() + t = self.now() + if t - self.heard > self.dead: + raise Refused(f'no word from the desktop in {self.dead} s') + if t - self.heard > self.ping * (self.pings + 1): + self.pings += 1 + self.send({'v': wire.VERSION, 'ping': self.pings}) + for client, (_, deadline) in list(self.waiting.items()): + if t > deadline: + self.drop(client) + for ident, (_, verb, deadline) in list(self.pending.items()): + if t > deadline: + self.reply(ident, wire.answer(wire.UNKNOWN, stderr=f'fork-fleet-desk: pool {self.pool}: {verb}: ' + 'no answer in time: it may or may not have run\n')) + + def from_agent(self, frame): + if 'pong' in frame: + wire.check_ping(frame, 'pong') + return + got = wire.check_answer(frame, with_id=True) + ident = got.pop('id') + if ident in self.pending: + self.reply(ident, got) + elif ident >= self.next_id: + raise wire.WireError(f'an answer to request {ident}, never sent') + + def accept(self, listener): + try: + client, _ = listener.accept() + except BlockingIOError: + return + uid = struct.unpack('3i', client.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))[1] + if uid not in (0, os.geteuid()): + client.close() + return + client.setblocking(False) + self.waiting[client] = (b'', self.now() + CLIENT) + self.sel.register(client, selectors.EVENT_READ, 'client') + + def from_client(self, client): + buf, deadline = self.waiting[client] + try: + data = client.recv(65536) + except BlockingIOError: + return + buf += data + if b'\n' not in buf: + if data and len(buf) < wire.MAX_FRAME: + self.waiting[client] = (buf, deadline) + else: + self.drop(client) + return + self.sel.unregister(client) + del self.waiting[client] + line, rest = buf.split(b'\n', 1) + try: + frame = wire.check_request(wire.decode(line), with_id=False) + if rest: + raise wire.WireError('more than one request') + except wire.WireError as e: + self.answer_client(client, wire.answer(2, stderr=f'fork-fleet-desk: refused: {e}\n')) + return + ident, self.next_id = self.next_id, self.next_id + 1 + self.pending[ident] = (client, frame['verb'], self.now() + wire.seconds(frame['verb'])) + self.send({**frame, 'id': ident}) + + def reply(self, ident, frame): + client = self.pending.pop(ident)[0] + self.answer_client(client, frame) + + def answer_client(self, client, frame): + try: + client.setblocking(True) + client.settimeout(CLIENT) + client.sendall(wire.encode(frame)) + except OSError: + pass # the client gave up (its own time ran out) + client.close() + + def drop(self, client): + self.sel.unregister(client) + del self.waiting[client] + client.close() + + +# ---------------------------------------------------------------- launch + +def launch(pool, verb, args, stdin, path, out=sys.stdout, err=sys.stderr, wait=None): + """The exit code; the desktop's stdout and stderr written to out and err.""" + pool_ok(pool) + try: + frame = wire.request(verb, args, stdin if verb == 'start' else None) + except wire.WireError as e: + raise Refused(f'refused: {e}', 2) from None + no_session = Refused(f'pool {pool}: no live desk session (the desktop is off, asleep or not linked): ' + f'{verb} was not sent', wire.NO_SESSION) + sock_path = os.path.join(path, f'{pool}.sock') + try: + owner = safe_dir(path, (0, os.geteuid())) + st = os.lstat(sock_path) + except FileNotFoundError: + raise no_session from None + if not stat.S_ISSOCK(st.st_mode) or st.st_uid != owner: + raise Refused(f'{sock_path} is not a socket of uid {owner}') + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: + s.settimeout(CLIENT) + try: + s.connect(sock_path) + except (ConnectionRefusedError, FileNotFoundError): + raise no_session from None + except OSError as e: + raise Refused(f'cannot reach {sock_path}: {e.strerror}') from None + unknown = Refused(f'pool {pool}: {verb}: no answer from the desktop: it may or may not have run', wire.UNKNOWN) + try: + s.sendall(wire.encode(frame)) + deadline = time.monotonic() + (wire.seconds(verb) if wait is None else wait) + buf = b'' + while b'\n' not in buf: + left = deadline - time.monotonic() + if left <= 0: + raise unknown + s.settimeout(left) + data = s.recv(65536) + if not data or len(buf) > wire.MAX_FRAME: + raise unknown + buf += data + got = wire.check_answer(wire.decode(buf.split(b'\n', 1)[0]), with_id=False) + except (OSError, wire.WireError): + raise unknown from None + out.write(got['stdout']) + err.write(got['stderr']) + return got['exit'] + + +def main(argv=None, env=os.environ): + argv = sys.argv[1:] if argv is None else argv + try: + if len(argv) == 2 and argv[0] == 'serve': + for sig in (signal.SIGTERM, signal.SIGHUP): + signal.signal(sig, lambda *_: sys.exit(1)) # through serve's finally: the socket goes, clients learn + return serve(argv[1], sock_dir(env)) + if len(argv) >= 3 and argv[0] == 'launch': + stdin = sys.stdin.read(wire.MAX_FRAME) if argv[2] == 'start' else None + return launch(argv[1], argv[2], argv[3:], stdin, sock_dir(env)) + raise Refused('usage: serve POOL | launch POOL VERB ARGS...', 2) + except Refused as e: + print(f'fork-fleet-desk: {e}', file=sys.stderr) + return e.code + except OSError as e: + print(f'fork-fleet-desk: {e.filename or ""}: {e.strerror}', file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/fork-fleet-desk.test.py b/.github/scripts/fork-fleet-desk.test.py new file mode 100644 index 0000000000000..dee31ea0de964 --- /dev/null +++ b/.github/scripts/fork-fleet-desk.test.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Tests of fork-fleet-desk.py (the desk link's always-on side) and of fork-fleet-desk-wire.py, its wire format, with +the contract between the wire and fork-fleet-launch.sh and fork-fleet.py. A fake desk agent on pipes, real Unix sockets +in a temp directory; no root, no ssh, no network. Run: python3 . Linux only.""" +import importlib.util, inspect, io, os, re, signal, subprocess, sys, tempfile, threading, time, unittest + +sys.dont_write_bytecode = True +HERE = os.path.dirname(os.path.abspath(__file__)) +DESK = os.path.join(HERE, 'fork-fleet-desk.py') + + +def load(name, file): + spec = importlib.util.spec_from_file_location(name, os.path.join(HERE, file)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +desk, ff = load('fork_fleet_desk', 'fork-fleet-desk.py'), load('fork_fleet', 'fork-fleet.py') +wire = desk.wire +POOL = 'desk' +START_IN = 'ACTIONS_RUNNER_INPUT_JITCONFIG=abc+/=\nFORK_FLEET_RUN=7\nFORK_FLEET_ATTEMPT=1\n' + + +class Agent: + """A fake desk agent: says hello, answers pings, and answers each request with `answer(frame)` (None: silence).""" + + def __init__(self, answer, hello=POOL, pong=True): + self.answer, self.pong, self.got = answer, pong, [] + self.to_serve_r, self.to_serve_w = os.pipe() + self.from_serve_r, self.from_serve_w = os.pipe() + os.write(self.to_serve_w, wire.encode({'v': 1, 'hello': hello})) + threading.Thread(target=self.run, daemon=True).start() + + def run(self): + reader = wire.Reader() + while data := os.read(self.from_serve_r, 65536): + for frame in reader.feed(data): + self.got.append(frame) + if 'ping' in frame: + reply = {'v': 1, 'pong': frame['ping']} if self.pong else None + elif 'id' in frame: + reply = self.answer(frame) + reply = reply and {**reply, 'id': frame['id']} + else: + continue + if reply is not None: + os.write(self.to_serve_w, wire.encode(reply)) + + def hang_up(self): + os.close(self.to_serve_w) + + +class Link(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = os.path.join(self.tmp.name, 'run') + self.sock = os.path.join(self.dir, f'{POOL}.sock') + + def tearDown(self): + self.tmp.cleanup() + + def serve(self, agent, **kw): + log = io.StringIO() + result = {} + t = threading.Thread(target=lambda: result.update(code=desk.serve( + POOL, self.dir, agent.to_serve_r, agent.from_serve_w, log=log, **kw)), daemon=True) + t.start() + for _ in range(200): + if os.path.exists(self.sock) or not t.is_alive(): + break + time.sleep(0.01) + return t, result, log + + def launch(self, verb, *args, stdin=None, wait=None): + out, err = io.StringIO(), io.StringIO() + try: + code = desk.launch(POOL, verb, list(args), stdin, self.dir, out, err, wait) + except desk.Refused as e: + code, err = e.code, io.StringIO(str(e)) + return code, out.getvalue(), err.getvalue() + + def test_verbs_round_trip_with_exit_codes_and_output(self): + def answer(f): + if f['verb'] == 'active': + return wire.answer(0, 'fleet-desk-1\n', '') + if f['verb'] == 'start': + return wire.answer(0) if f['stdin'] == START_IN else wire.answer(9) + return wire.answer(1, '', 'not enough memory\x1b[2J\n') + agent = Agent(answer) + t, _, _ = self.serve(agent) + self.assertEqual(self.launch('active'), (0, 'fleet-desk-1\n', '')) + self.assertEqual(self.launch('start', 'fleet-desk-5', stdin=START_IN)[0], 0) + self.assertEqual(self.launch('fits', '4096'), (1, '', 'not enough memory?[2J\n')) # escapes made harmless + self.assertEqual(oct(os.stat(self.sock).st_mode & 0o777), '0o600') + self.assertEqual(oct(os.stat(self.dir).st_mode & 0o777), '0o700') + agent.hang_up() + t.join(5) + self.assertFalse(t.is_alive()) + self.assertFalse(os.path.exists(self.sock)) + + def test_no_session_fails_at_once_and_sends_nothing(self): + began = time.monotonic() + self.assertEqual(self.launch('active')[0], wire.NO_SESSION) # no directory at all (after a boot) + os.mkdir(self.dir, 0o700) + self.assertEqual(self.launch('active')[0], wire.NO_SESSION) # no socket + import socket + with socket.socket(socket.AF_UNIX) as s: # a socket left by a killed session + s.bind(self.sock) + code, _, err = self.launch('active') + self.assertEqual(code, wire.NO_SESSION) + self.assertIn('no live desk session', err) + self.assertLess(time.monotonic() - began, 1) + + def test_bad_requests_are_refused_before_sending(self): + agent = Agent(lambda f: wire.answer(0)) + t, _, _ = self.serve(agent) + for verb, args, stdin in (('start', ['fleet-desk-5'], START_IN + 'EVIL=1\n'), ('start', ['../x-1'], START_IN), + ('stop', ['a b-1'], None), ('rm', ['-rf'], None), ('active', ['x'], None), + ('fits', ['1e9'], None), ('check-net', ['file:///etc', '1.2.3.4:5'], None)): + self.assertEqual(self.launch(verb, *args, stdin=stdin)[0], 2, verb) + self.assertEqual([f for f in agent.got if 'id' in f], []) + agent.hang_up() + t.join(5) + + def test_second_session_for_the_pool_is_refused(self): + a = Agent(lambda f: wire.answer(0)) + t, _, _ = self.serve(a) + b, got = Agent(lambda f: wire.answer(0)), [] + + def second(): + try: + desk.serve(POOL, self.dir, b.to_serve_r, b.from_serve_w, log=io.StringIO()) + except desk.Refused as e: + got.append(e.code) + t2 = threading.Thread(target=second, daemon=True) + t2.start() + t2.join(5) + b.hang_up() + self.assertEqual(got, [wire.NO_SESSION]) + self.assertEqual([f for f in b.got if 'hello' in f], []) # refused before the hello + self.assertEqual(self.launch('active')[0], 0) # the first one still serves + a.hang_up() + t.join(5) + + def test_wrong_hello_or_bad_frame_ends_the_session(self): + t, result, log = self.serve(Agent(lambda f: wire.answer(0), hello='other')) + t.join(5) + self.assertEqual(result['code'], 1) + self.assertIn('hello for pool', log.getvalue()) + self.assertFalse(os.path.exists(self.sock)) + agent = Agent(lambda f: {'v': 1, 'exit': 0, 'stdout': '', 'stderr': '', 'extra': 1}) + t, result, log = self.serve(agent) + self.assertEqual(self.launch('active')[0], wire.UNKNOWN) + t.join(5) + self.assertEqual(result['code'], 1) + self.assertEqual(self.launch('active')[0], wire.NO_SESSION) + + def test_desktop_gone_mid_verb_answers_unknown_at_once(self): + agent = Agent(lambda f: None) + t, _, _ = self.serve(agent) + threading.Timer(0.3, agent.hang_up).start() + began = time.monotonic() + code, _, err = self.launch('start', 'fleet-desk-5', stdin=START_IN, wait=20) + self.assertEqual(code, wire.UNKNOWN) + self.assertIn('the desk session ended', err) + self.assertLess(time.monotonic() - began, 3) + t.join(5) + self.assertEqual(self.launch('active')[0], wire.NO_SESSION) + + def test_silent_desktop_ends_the_session(self): + agent = Agent(lambda f: None, pong=False) + t, result, log = self.serve(agent, ping=0.2, dead=0.6) + t.join(5) + self.assertEqual(result['code'], 1) + self.assertIn('no word from the desktop', log.getvalue()) + self.assertTrue(any('ping' in f for f in agent.got)) + self.assertEqual(self.launch('active')[0], wire.NO_SESSION) + + def test_answering_pings_keeps_the_session_and_slow_answer_is_unknown(self): + agent = Agent(lambda f: None) + t, _, _ = self.serve(agent, ping=0.2, dead=0.6) + code, _, _ = self.launch('active', wait=1.5) # no answer: unknown, not a hang + self.assertEqual(code, wire.UNKNOWN) + self.assertTrue(t.is_alive()) # pongs kept it open past `dead` + agent.hang_up() + t.join(5) + + def test_socket_directory_must_be_private(self): + os.mkdir(self.dir, 0o755) + with self.assertRaises(desk.Refused): + desk.serve(POOL, self.dir, *os.pipe(), log=io.StringIO()) + self.assertNotEqual(self.launch('active')[0], 0) + os.rmdir(self.dir) + os.symlink(self.tmp.name, self.dir) + with self.assertRaises(desk.Refused): + desk.safe_dir(self.dir, (os.geteuid(),)) + + def test_only_root_uses_the_fixed_directory(self): + env = {'FORK_FLEET_DESK_DIR': '/tmp/elsewhere'} + self.assertEqual(desk.sock_dir(env, euid=0), desk.DIR) + self.assertEqual(desk.sock_dir(env, euid=1000), '/tmp/elsewhere') + self.assertEqual(desk.sock_dir({}, euid=1000), desk.DIR) + + def test_cli_and_the_dispatcher_through_the_link(self): + agent = Agent(lambda f: wire.answer(0, 'fleet-desk-1\nfleet-desk-2\nother-3\n') if f['verb'] == 'active' + else wire.answer(1, '', 'no\n')) + t, _, _ = self.serve(agent) + env = {'PATH': os.environ['PATH'], 'FORK_FLEET_DESK_DIR': self.dir} + got = subprocess.run([sys.executable, '-I', DESK, 'launch', POOL, 'active'], env=env, capture_output=True, + text=True, timeout=30) + self.assertEqual((got.returncode, got.stdout), (0, 'fleet-desk-1\nfleet-desk-2\nother-3\n')) + pool = {'name': POOL, 'label': 'fleet-desk', 'launch': [sys.executable, '-I', DESK, 'launch', POOL]} + host = ff.Host(pool, run=lambda *a, **kw: subprocess.run(*a, **kw, env=env)) + self.assertEqual(host.units(), {'fleet-desk-1', 'fleet-desk-2'}) + self.assertFalse(host.fits(4096)) + with self.assertRaises(ff.HostError) as e: + host.stop('fleet-desk-1') + self.assertIn('no', str(e.exception)) + agent.hang_up() + t.join(5) + with self.assertRaises(ff.HostError) as e: + host.units() + self.assertIn('no live desk session', str(e.exception)) + + def test_serve_cli_cleans_up_on_sigterm(self): + env = {'PATH': os.environ['PATH'], 'FORK_FLEET_DESK_DIR': self.dir} + p = subprocess.Popen([sys.executable, '-I', DESK, 'serve', POOL], env=env, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.stdin.write(wire.encode({'v': 1, 'hello': POOL})) + p.stdin.flush() + self.assertEqual(wire.decode(p.stdout.readline().rstrip(b'\n')), {'v': 1, 'hello': POOL}) + for _ in range(200): + if os.path.exists(self.sock): + break + time.sleep(0.01) + p.send_signal(signal.SIGTERM) + p.wait(10) + p.stdin.close(), p.stdout.close(), p.stderr.close() + self.assertFalse(os.path.exists(self.sock)) + self.assertEqual(self.launch('active')[0], wire.NO_SESSION) + + +class Wire(unittest.TestCase): + """The wire's rules, and its contract with fork-fleet-launch.sh and fork-fleet.py (one source each).""" + LAUNCH = open(os.path.join(HERE, 'fork-fleet-launch.sh'), encoding='utf-8').read() + + def shell_var(self, name): + return re.search(rf"^{name}='\^(.*)\$'$", self.LAUNCH, re.M).group(1) + + def test_verbs_are_the_launchers(self): + case = self.LAUNCH[self.LAUNCH.index('case "${1-}:$#" in'):] + verbs = dict(re.findall(r'^ ([a-z-]+):([0-9]+|\*)\)', case, re.M)) + self.assertEqual(set(verbs), set(wire.VERBS)) + for verb, n in verbs.items(): + fixed, rest, _ = wire.VERBS[verb] + self.assertEqual(n == '*', rest is not None, verb) + if n != '*': + self.assertEqual(int(n) - 1, len(fixed), verb) + + def test_patterns_are_the_launchers_and_the_dispatchers(self): + self.assertEqual(wire.NAME.pattern, self.shell_var('NAME')) + self.assertEqual(wire.RUNNER_VERSION.pattern, self.shell_var('VERSION')) + self.assertEqual(wire.POOL.pattern, ff.POOL.pattern) + self.assertIn(f'$url =~ ^{wire.URL.pattern}$ ]]', self.LAUNCH) + + def test_every_dispatcher_verb_answers_before_the_dispatcher_gives_up(self): + limit = int(re.search(r'timeout=([0-9]+)', inspect.getsource(ff.Host._do)).group(1)) + for verb in ('start', 'ready', 'active', 'fits', 'stop', 'prune'): + self.assertLess(wire.seconds(verb), limit, verb) + + def test_the_dispatchers_start_input_passes(self): + text = ff.runner_env('abc+/=', {'id': 123, 'run_attempt': 2}) + wire.check_start_input(text) + for bad in (text + 'X=1\n', text.replace('FORK_FLEET_RUN=123\n', ''), text + text, text.replace('123', '0')): + with self.assertRaises(wire.WireError): + wire.check_start_input(bad) + + def test_frames(self): + r = wire.Reader() + frame = wire.request('start', ['x-1'], START_IN, ident=4) + self.assertEqual(r.feed(wire.encode(frame)[:5]), []) + self.assertEqual(r.feed(wire.encode(frame)[5:]), [frame]) + for bad in (b'{"v":2}\n', b'[1]\n', b'nope\n', b'{"v":true}\n', b'\xff\n', b'x' * (wire.MAX_FRAME + 1)): + with self.assertRaises(wire.WireError, msg=bad[:20]): + wire.Reader().feed(bad) + for bad in ({'v': 1, 'id': 1, 'exit': 256, 'stdout': '', 'stderr': ''}, + {'v': 1, 'id': -1, 'exit': 0, 'stdout': '', 'stderr': ''}, + {'v': 1, 'id': 1, 'exit': True, 'stdout': '', 'stderr': ''}, + {'v': 1, 'id': 1, 'exit': 0, 'stdout': 'x' * (wire.MAX_TEXT + 1), 'stderr': ''}): + with self.assertRaises(wire.WireError): + wire.check_answer(bad, with_id=True) + with self.assertRaises(wire.WireError): + wire.check_request({'v': 1, 'verb': 'active', 'args': [], 'stdin': 'x'}, with_id=False) + + +if __name__ == '__main__': + unittest.main() From f30564d5bf36f76c2f673dd94f37bf7f380ad177 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:49:09 +0000 Subject: [PATCH 2/2] desk link: dedicated user, socket group, byte sizes, time budget (fix round 1) - serve runs as a dedicated unprivileged user (default fleet-desk) and refuses root; the socket is 0660 in the directory's group, made by a systemd-tmpfiles line; the peer check lets in root, its own user and the socket group. A lock or directory left by an older root serve is refused with a clear message. - wire: sizes in bytes (fit() cuts answers), deep nesting is a WireError, lone surrogates are made harmless, a wrong version exits 76 and says which side to upgrade, start input must end in a newline, and the time budget (limit < serve < launch < 120 s) is stated and tested; the agent must kill a verb at its limit. - launch: one overall deadline, connect bounded, a launcher's own exit 3 becomes 1, odd bytes refused with exit 2 and no traceback. - serve: one accept() error costs one client, not the session. - dispatcher: a pool down for the same reason is logged once, then every 10 minutes with a count, then once when it answers again. - runbook: the desk-link user (/bin/sh, no password), tmpfiles line, install of both desk files, one dispatcher per desk pool, upgrading from a root-run link. - tests: the reviewer's 7 extra tests adopted, plus new ones; all 34 guard mutations are caught. Co-Authored-By: Claude Opus 5.5 --- .github/fork-fleet.md | 71 ++++-- .github/scripts/fork-fleet-desk-wire.py | 97 ++++++-- .github/scripts/fork-fleet-desk.py | 139 ++++++++--- .github/scripts/fork-fleet-desk.test.py | 294 +++++++++++++++++++++++- .github/scripts/fork-fleet.py | 26 ++- .github/scripts/fork-fleet.test.py | 17 ++ 6 files changed, 575 insertions(+), 69 deletions(-) diff --git a/.github/fork-fleet.md b/.github/fork-fleet.md index 69824fc5e59aa..2ff3bf2ff44d6 100644 --- a/.github/fork-fleet.md +++ b/.github/fork-fleet.md @@ -386,7 +386,7 @@ done echo "template: actions/runner $(cat "$live/.fork-fleet-version")" ) -# 3. The fleet files, from one pinned main commit. All four are fetched first and installed only when each came; +# 3. The fleet files, from one pinned main commit. All six are fetched first and installed only when each came; # SOURCE is written last, so it never names a commit whose files are not all in place. ( set -euo pipefail @@ -395,7 +395,8 @@ done tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT base=https://raw.githubusercontent.com/scuttletoby/t3code/$sha/.github files="fork-fleet.json:fork-fleet.json:644 scripts/fork-fleet.py:fork-fleet.py:644 - scripts/fork-fleet-hook.sh:hook.sh:755 scripts/fork-fleet-launch.sh:fork-fleet-launch.sh:755" + scripts/fork-fleet-hook.sh:hook.sh:755 scripts/fork-fleet-launch.sh:fork-fleet-launch.sh:755 + scripts/fork-fleet-desk.py:fork-fleet-desk.py:644 scripts/fork-fleet-desk-wire.py:fork-fleet-desk-wire.py:644" for f in $files; do IFS=: read -r src dst mode <<<"$f" curl -fsSLo "$tmp/$dst" "$base/$src" @@ -404,6 +405,8 @@ done out=$(python3 -I "$tmp/fork-fleet.py" --config "$tmp/fork-fleet.json" gate &1 || true) grep -q 'no Runner.Worker above this hook' <<<"$out" || { printf '%s\n' "$out"; exit 1; } bash -n "$tmp/hook.sh" && bash -n "$tmp/fork-fleet-launch.sh" + out=$(python3 -I "$tmp/fork-fleet-desk.py" 2>&1 || true) # the desk link loads its wire format + grep -q 'usage: serve POOL' <<<"$out" || { printf '%s\n' "$out"; exit 1; } install -d -m 755 /opt/fork-fleet for f in $files; do IFS=: read -r src dst mode <<<"$f" @@ -467,23 +470,59 @@ that runs `fork-fleet-launch.sh` on the PC over ssh. Run the dispatcher with ### The desk link (a desktop pool) A desktop accepts no connection and holds no GitHub token. Its desk agent dials -out to the always-on host over ssh, with a key that may only run `serve` for -its own pool, and the pool's dispatcher on the always-on host sends the -launcher's verbs through that one session (`fork-fleet-desk.py`, its wire -format `fork-fleet-desk-wire.py`): +out to the always-on host over ssh, as a dedicated unprivileged user (default +`fleet-desk`, never root), with a key that may only run `serve` for its own +pool. The pool's dispatcher on the always-on host sends the launcher's verbs +through that one session (`fork-fleet-desk.py`, its wire format +`fork-fleet-desk-wire.py`). Once per always-on host, as root (step 3 installs +both files): -``` -# root's authorized_keys on the always-on host (sshd: PermitRootLogin forced-commands-only), one line per pool: -restrict,command="python3 -I /opt/fork-fleet/fork-fleet-desk.py serve POOL" ssh-ed25519 AAAA... desk-POOL -# the pool's launch command in the project's fleet config: -"launch": ["python3", "-I", "/opt/fork-fleet/fork-fleet-desk.py", "launch", "POOL"] +```sh +# The desk-link user: a real shell (sshd runs the forced command through it, so nologin would close the link) and no +# password ('*': none, but not locked; a locked '!' account can be refused by sshd). Its key lines limit it. +useradd --system --create-home --home-dir /var/lib/fleet-desk --shell /bin/sh fleet-desk +usermod -p '*' fleet-desk +# The socket directory, at every boot: owned by that user, its group is the socket group (who may connect). +echo 'd /run/fork-fleet-desk 0710 fleet-desk fleet-desk -' > /etc/tmpfiles.d/fork-fleet-desk.conf +systemd-tmpfiles --create /etc/tmpfiles.d/fork-fleet-desk.conf +# Per desk pool: the desktop's public key, as the desk-link user, with the key tool fork-fleet-desk-key.py +# (row D4 of the desktop-pool design; install it beside the others). +sudo -u fleet-desk -H python3 -I /opt/fork-fleet/fork-fleet-desk-key.py add POOL < desk-POOL.pub ``` -`serve` keeps one session per pool (a lock), listens on a root-only socket in -`/run/fork-fleet-desk`, and pings the agent; `launch` fails at once (exit 3) -when no session is live, so a sleeping desktop's pool is down for the pass and -`route` sends its jobs elsewhere. A verb with no answer in its time exits 75: -it may or may not have run, and the next pass's `active` tells. +The line it writes (and the only kind of line that user's key file holds) is +`restrict,command="python3 -I /opt/fork-fleet/fork-fleet-desk.py serve POOL" ssh-ed25519 AAAA... fork-fleet-desk:POOL` +in `~fleet-desk/.ssh/authorized_keys`. The pool's launch command in the +project's fleet config is +`"launch": ["python3", "-I", "/opt/fork-fleet/fork-fleet-desk.py", "launch", "POOL"]`. +Another user name: use it in all three places above, and give the dispatcher +`Environment=FORK_FLEET_DESK_USER=NAME`. Another socket group: the group in the +tmpfiles line; the desk-link user must be in it, and so must the dispatcher's +user when that is not root. + +`serve` keeps one session per pool (a lock), listens on a 0660 socket in +`/run/fork-fleet-desk` (it lets in only root, its own user and the socket +group), and pings the agent; `launch` fails at once (exit 3) when no session is +live, so a sleeping desktop's pool is down for the pass and `route` sends its +jobs elsewhere. The dispatcher says so once, then every 10 minutes with a count, +and once more when the desktop answers again. A verb with no answer in its time +exits 75: it may or may not have run, and the next pass's `active` tells. + +**One dispatcher per desk pool.** Run each desk pool's dispatcher as its own +unit (a copy of step 5's unit named `t3code-fleet-dispatcher-POOL`, with +`dispatch --pool POOL`), never in the vm pool's unit: one dispatcher serves its +pools one after another, and a desktop that answers each verb just inside its +time could hold a pass for minutes, so the vm pool would wait behind the least +trusted machine. A second dispatcher for a pool that one already serves refuses +to start (the lock `/run/fork-fleet/dispatch-POOL.lock`) and its unit keeps +failing every 30 s: stop it. + +**From an older desk link** (serve as root, the key in root's +authorized_keys): remove that key line and `PermitRootLogin +forced-commands-only`, do the steps above, and restart the host or remove the +old `/run/fork-fleet-desk` first (`serve` refuses a lock file or directory left +by root, with a message that says so). A desk agent of an older wire version +gets exit 76 and a line that says to upgrade it. **What jobs need on the host.** The jobs run without sudo, so the host must already have: diff --git a/.github/scripts/fork-fleet-desk-wire.py b/.github/scripts/fork-fleet-desk-wire.py index 5c4fc4ffffd67..38f5db064bc61 100644 --- a/.github/scripts/fork-fleet-desk-wire.py +++ b/.github/scripts/fork-fleet-desk-wire.py @@ -1,24 +1,39 @@ -"""Fork only: the wire format of the desk link (.github/fork-fleet.md, "The desk link"), the one module both of its +r"""Fork only: the wire format of the desk link (.github/fork-fleet.md, "The desk link"), the one module both of its sides load: `fork-fleet-desk.py` on the always-on host (`serve`, `launch`) and the desk agent on the desktop. The link: the desktop dials out to the always-on host over ssh with a key that may only run `fork-fleet-desk.py serve POOL`. Through that session the always-on host's `launch POOL VERB ARGS...` (a pool's `launch` command for `fork-fleet.py dispatch`) has the desktop run `fork-fleet-launch.sh VERB ARGS...`. -Frames: one JSON object per line (ASCII, "\n"-terminated, at most MAX_FRAME bytes), each with "v": VERSION. +Frames: one JSON object per line, each with "v": VERSION. Sizes are in BYTES: a frame is ASCII (JSON's \uXXXX +escapes for anything else) and at most MAX_FRAME bytes with its "\n"; the stdout and stderr of an answer are each at +most MAX_TEXT bytes as written in the frame (text_size; `fit` cuts a text to that, `answer` uses it). Too long is a +WireError, never a cut on the receiving side. agent -> serve, first {"v":1, "hello": POOL} the pool the agent serves (must be serve's) serve -> agent, answer {"v":1, "hello": POOL} serve -> agent {"v":1, "id": N, "verb": V, "args": [...], "stdin": S} one launcher call ("stdin": start only) agent -> serve {"v":1, "id": N, "exit": E, "stdout": O, "stderr": R} its result; any order, once per id serve -> agent {"v":1, "ping": N} the agent answers {"v":1, "pong": N} at once, also while verbs run launch -> serve (socket) a request without "id"; serve answers with an answer without "id". -Anything else ends the session (or drops that one socket client). Standard library only. +Anything else ends the session (or drops that one socket client). A hello of another VERSION ends it with serve's +exit WRONG_VERSION (ssh passes it to the agent) and a line on ssh's stderr that says which side to upgrade. + +Time (the dispatcher, fork-fleet.py Host, gives a launch command 120 s): + limit(verb) VERBS' seconds. The agent MUST kill a verb still running at its limit and answer it UNKNOWN, and + MUST kill every verb still running when the session ends: a `start` left running after its 75 + could come up after the next pass deleted its registration. + serve_seconds limit + SERVE_MARGIN: serve answers UNKNOWN itself when the agent has not answered by then. + seconds(verb) limit + MARGIN: `launch`'s whole time, connect and send included; it then exits UNKNOWN. + The longest verb the dispatcher calls (start, stop, prune) ends by 110 s, under its 120 s. +Exit codes of `launch`: the launcher's own (0-255), except NO_SESSION (3: nothing was sent; a launcher's own 3 is +turned into 1), UNKNOWN (75: the verb may or may not have run; the agent answers 75 only for a verb it killed, and +answers a launcher's own 75 as 1), 2 (refused before sending) and 1 (any other failure). Standard library only. """ import json, re VERSION = 1 MAX_FRAME = 256 * 1024 -MAX_TEXT = 64 * 1024 # stdout or stderr of one answer +MAX_TEXT = 64 * 1024 # bytes of the stdout or the stderr of one answer, as written in the frame POOL = re.compile(r'[a-z0-9][a-z0-9_-]{0,39}') # fork-fleet.py's POOL NAME = re.compile(r'[a-z0-9][a-z0-9-]{0,62}-[0-9]{1,20}') # fork-fleet-launch.sh's NAME RUNNER_VERSION = re.compile(r'[0-9]{1,5}\.[0-9]{1,5}\.[0-9]{1,5}') # fork-fleet-launch.sh's VERSION @@ -26,8 +41,8 @@ URL = re.compile(r'https?://[A-Za-z0-9.-]+(:[0-9]{1,5})?(/[A-Za-z0-9._~/-]*)?') HOST_PORT = re.compile(r'[0-9]{1,3}(\.[0-9]{1,3}){3}:[0-9]{1,5}') START_LINE = re.compile(r'(ACTIONS_RUNNER_INPUT_JITCONFIG=[A-Za-z0-9+/=]{1,65536}|FORK_FLEET_(RUN|ATTEMPT)=[1-9][0-9]{0,19})') -# The launcher's verbs: (argument patterns, then a pattern every further argument matches or None, seconds the desktop -# may take). fork-fleet.py's Host waits 120 s for its launch command: every verb it calls answers before that. +# The launcher's verbs: (argument patterns, then a pattern every further argument matches or None, the verb's limit +# in seconds: the agent kills it then). fork-fleet.py's Host waits 120 s for its launch command: see "Time" above. VERBS = { 'start': ((NAME,), None, 100), 'ready': ((RUNNER_VERSION,), None, 30), @@ -37,16 +52,27 @@ 'prune': ((), None, 100), 'check-net': ((URL, HOST_PORT), HOST_PORT, 300), } -MARGIN = 10 # seconds `launch` waits beyond a verb's own, for the link itself +MARGIN = 10 # seconds `launch` waits beyond a verb's limit, all told +SERVE_MARGIN = 5 # seconds `serve` waits beyond it before it answers UNKNOWN itself UNKNOWN = 75 # launch's exit when the verb may or may not have run (EX_TEMPFAIL) NO_SESSION = 3 # launch's exit when no desk session is live: nothing was sent -CONTROL = re.compile(r'[\x00-\x08\x0b-\x1f\x7f-\x9f]') +WRONG_VERSION = 76 # serve's exit on a frame of another VERSION (EX_PROTOCOL): one side needs an upgrade +CONTROL = re.compile('[\x00-\x08\x0b-\x1f\x7f-\x9f\ud800-\udfff]') # also lone surrogates: they cannot be printed class WireError(ValueError): pass +class VersionError(WireError): + """A frame of another VERSION: `got` is its "v".""" + + def __init__(self, got): + side = 'the desktop agent' if got < VERSION else "this host's fork-fleet files" + super().__init__(f'a frame of wire version {got}, this side speaks {VERSION}: upgrade {side}') + self.got = got + + def encode(frame): data = json.dumps(frame, separators=(',', ':'), ensure_ascii=True).encode() + b'\n' if len(data) > MAX_FRAME: @@ -78,8 +104,12 @@ def decode(line): frame = json.loads(line.decode('ascii')) except (UnicodeDecodeError, ValueError): raise WireError('not a JSON line') from None - if not isinstance(frame, dict) or frame.get('v') != VERSION or type(frame['v']) is not int: + except RecursionError: + raise WireError('not a JSON line: nested too deep') from None + if not isinstance(frame, dict) or type(frame.get('v')) is not int: raise WireError(f'not a version {VERSION} frame') + if frame['v'] != VERSION: + raise VersionError(frame['v']) return frame @@ -107,10 +137,11 @@ def check_ping(frame, key): def check_start_input(text): - """The launcher's `start` input: its three lines, each once, nothing else.""" - lines = text.split('\n') - if lines and lines[-1] == '': - lines.pop() + """The launcher's `start` input: its three lines, each once and ending in "\n" (the launcher's `read` drops an + unended last line), nothing else.""" + if not text.endswith('\n'): + raise WireError('start: the input does not end in a newline') + lines = text.split('\n')[:-1] keys = [line.split('=', 1)[0] for line in lines] if sorted(keys) != ['ACTIONS_RUNNER_INPUT_JITCONFIG', 'FORK_FLEET_ATTEMPT', 'FORK_FLEET_RUN'] or \ not all(START_LINE.fullmatch(line) for line in lines): @@ -148,18 +179,48 @@ def check_request(frame, with_id): return frame +def limit(verb): + """When the agent kills `verb` (still running) and answers UNKNOWN.""" + return VERBS[verb][2] + + +def serve_seconds(verb): + """How long `serve` waits for the agent's answer to `verb` before it answers UNKNOWN itself.""" + return VERBS[verb][2] + SERVE_MARGIN + + def seconds(verb): - """How long `launch` waits for an answer to `verb`.""" + """How long `launch` takes for `verb` at most, all told (connect, send, the answer).""" return VERBS[verb][2] + MARGIN def clean(text): - """Text shown to a person or a log: no control characters but newline and tab.""" + """Text shown to a person or a log: no control characters but newline and tab, no lone surrogates.""" return CONTROL.sub('?', text) +def text_size(text): + """Bytes `text` takes in a frame.""" + return len(json.dumps(text, ensure_ascii=True)) - 2 + + +CUT = '\n[cut to fit the desk link]\n' + + +def fit(text, most=MAX_TEXT): + """`text`, or its longest start that fits in `most` bytes of a frame, then CUT.""" + if text_size(text) <= most: + return text + lo, hi = 0, len(text) # text[:lo] + CUT fits, text[:hi] + CUT does not + while hi - lo > 1: + mid = (lo + hi) // 2 + lo, hi = (mid, hi) if text_size(text[:mid] + CUT) <= most else (lo, mid) + return text[:lo] + CUT + + def answer(exit_code, stdout='', stderr='', ident=None): - frame = {'v': VERSION, 'exit': exit_code, 'stdout': stdout, 'stderr': stderr} + """A checked answer frame; stdout and stderr are cut to fit (`fit`).""" + frame = {'v': VERSION, 'exit': exit_code, 'stdout': fit(stdout), 'stderr': fit(stderr)} if ident is not None: frame['id'] = ident return check_answer(frame, with_id=ident is not None) @@ -172,7 +233,7 @@ def check_answer(frame, with_id): if type(frame['exit']) is not int or not 0 <= frame['exit'] <= 255: raise WireError('answer: exit is not 0 to 255') for key in ('stdout', 'stderr'): - if not isinstance(frame[key], str) or len(frame[key]) > MAX_TEXT: - raise WireError(f'answer: {key} is not text of at most {MAX_TEXT} characters') + if not isinstance(frame[key], str) or text_size(frame[key]) > MAX_TEXT: + raise WireError(f'answer: {key} is not text of at most {MAX_TEXT} bytes') frame[key] = clean(frame[key]) return frame diff --git a/.github/scripts/fork-fleet-desk.py b/.github/scripts/fork-fleet-desk.py index 325b17237f3fb..b24d90f9b2df6 100755 --- a/.github/scripts/fork-fleet-desk.py +++ b/.github/scripts/fork-fleet-desk.py @@ -1,24 +1,30 @@ #!/usr/bin/env python3 -"""Fork only: the always-on host's side of the desk link (.github/fork-fleet.md, "The desk link"), as root. +"""Fork only: the always-on host's side of the desk link (.github/fork-fleet.md, "The desk link"). A desktop accepts no connection and holds no GitHub token: its desk agent dials out to the always-on host over ssh, -with a key that may only run `serve` for its own pool, and the pool's runners are started through that session. +as the dedicated, unprivileged desk-link user (USER), with a key that may only run `serve` for its own pool, and the +pool's runners are started through that session. python3 -I fork-fleet-desk.py serve POOL - The ssh forced command (authorized_keys: `restrict,command="python3 -I /opt/fork-fleet/fork-fleet-desk.py serve - POOL" ssh-ed25519 ...`). Takes DIR/POOL.lock (one session per pool: a second one exits 3 at once), reads the - agent's hello, then listens on the socket DIR/POOL.sock (0600 in a 0700 DIR, both this user's) and passes each + The ssh forced command, as the desk-link user, never root (its authorized_keys: `restrict,command="FORCED + POOL" ssh-ed25519 ...`, written by fork-fleet-desk-key.py). Takes DIR/POOL.lock (one session per pool: a + second one exits 3 at once), reads the agent's hello, then listens on the socket DIR/POOL.sock and passes each request to the agent and its answer back. Pings the agent every PING s; with no word from it in DEAD s, or on its end of the session, every request in flight is answered "outcome unknown" and the socket is removed. python3 -I fork-fleet-desk.py launch POOL VERB ARGS... A pool's `launch` command for `fork-fleet.py dispatch`: ["python3", "-I", ".../fork-fleet-desk.py", "launch", POOL]. Sends one launcher verb (`start` with its stdin) through the live session and prints the desktop's stdout and stderr and exits with its exit code. No live session: exit 3 at once, nothing sent. No answer in - the verb's time (fork-fleet-desk-wire.py VERBS): exit 75, the verb may or may not have run. -DIR is /run/fork-fleet-desk, root's. Not as root (tests), $FORK_FLEET_DESK_DIR may name another one. The link keeps no -state of its own: the live session is the state. Wire format: fork-fleet-desk-wire.py. Standard library only. + the verb's time (fork-fleet-desk-wire.py, "Time"): exit 75, the verb may or may not have run. +DIR is /run/fork-fleet-desk, made at boot by systemd-tmpfiles (runbook): owned by the desk-link user, its group the +socket group (who may connect: the dispatcher's user, or root), mode 0710 or 0750. The socket is 0660 in that group, +and serve lets in only root, its own user and members of that group (SO_PEERCRED). Not as root (tests), +$FORK_FLEET_DESK_DIR may name another DIR; for serve that only moves its own socket away from the dispatcher (sshd +passes no such variable unless its AcceptEnv names it). $FORK_FLEET_DESK_USER names the desk-link user for `launch` +(default USER). The link keeps no state of its own: the live session is the state. Wire format: +fork-fleet-desk-wire.py. Standard library only. """ -import importlib.util, os, selectors, signal, socket, stat, struct, sys, time +import grp, importlib.util, os, pwd, selectors, signal, socket, stat, struct, sys, time sys.dont_write_bytecode = True # a root-owned install directory gets no __pycache__ HERE = os.path.dirname(os.path.realpath(__file__)) @@ -27,6 +33,10 @@ _spec.loader.exec_module(wire) DIR = '/run/fork-fleet-desk' +USER = 'fleet-desk' # the desk-link user's default name: the same as fork-fleet-desk-key.py's USER +INSTALLED = '/opt/fork-fleet/fork-fleet-desk.py' +FORCED = f'python3 -I {INSTALLED} serve' # + POOL: the forced command, pinned by fork-fleet-desk-key.py's SERVE +TMPFILES = f'd {DIR} 0710 {USER} {USER} -' # the runbook's tmpfiles.d line (default user and socket group) HELLO = 30 # seconds the agent has for its hello PING, DEAD = 15, 45 CLIENT = 5 # seconds a socket client has to send its request, and to take its answer @@ -44,16 +54,30 @@ def sock_dir(env=os.environ, euid=None): def safe_dir(path, owners, make=False): - """The socket directory: not a symlink, owned by one of `owners`, no access for anyone else.""" + """The socket directory's stat: not a symlink, owned by one of `owners`, its group may at most pass through or + list it, anyone else nothing.""" if make: try: os.mkdir(path, 0o700) except FileExistsError: pass + except PermissionError: + raise Refused(f'cannot make {path}: make it at boot with systemd-tmpfiles ("{TMPFILES}", ' + '.github/fork-fleet.md, "The desk link")') from None st = os.lstat(path) - if not stat.S_ISDIR(st.st_mode) or st.st_uid not in owners or st.st_mode & 0o077: - raise Refused(f'{path} must be a directory of uid {owners[0]}, mode 0700') - return st.st_uid + if not stat.S_ISDIR(st.st_mode) or st.st_uid not in owners or st.st_mode & 0o027: + raise Refused(f'{path} must be a directory of uid {" or ".join(map(str, owners))}, mode 0710 or 0750 (an ' + f'older one, of root, 0700? the tmpfiles line "{TMPFILES}" fixes it at the next boot or ' + '`systemd-tmpfiles --create`)') + return st + + +def desk_uid(env): + """The uid of the desk-link user ($FORK_FLEET_DESK_USER, default USER), or None when there is no such user.""" + try: + return pwd.getpwnam(env.get('FORK_FLEET_DESK_USER') or USER).pw_uid + except KeyError: + return None def pool_ok(pool): @@ -65,28 +89,41 @@ def pool_ok(pool): def serve(pool, path, inp=0, out=1, log=sys.stderr, ping=PING, dead=DEAD, hello_wait=HELLO, now=time.monotonic): pool_ok(pool) - safe_dir(path, (os.geteuid(),), make=True) - lock = os.open(os.path.join(path, f'{pool}.lock'), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + if os.geteuid() == 0: + raise Refused(f'serve never runs as root: give the key to the desk-link user ({USER}), .github/fork-fleet.md, ' + '"The desk link"') + gid = safe_dir(path, (os.geteuid(),), make=True).st_gid + lock_path = os.path.join(path, f'{pool}.lock') + try: + lock = os.open(lock_path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600) + except PermissionError: + lock = None + if lock is None or os.fstat(lock).st_uid != os.geteuid(): + if lock is not None: + os.close(lock) + raise Refused(f'{lock_path} is not this user\'s (left by an older serve that ran as root?): remove it, or ' + 'restart the host') try: import fcntl try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: raise Refused(f'another desk session serves pool {pool}', wire.NO_SESSION) from None - return Session(pool, path, inp, out, log, ping, dead, hello_wait, now).run() + return Session(pool, path, inp, out, log, ping, dead, hello_wait, now, gid).run() finally: os.close(lock) class Session: - def __init__(self, pool, path, inp, out, log, ping, dead, hello_wait, now): - self.pool, self.inp, self.out, self.log, self.now = pool, inp, out, log, now + def __init__(self, pool, path, inp, out, log, ping, dead, hello_wait, now, gid): + self.pool, self.inp, self.out, self.log, self.now, self.gid = pool, inp, out, log, now, gid self.ping, self.dead, self.hello_wait = ping, dead, hello_wait self.sock_path = os.path.join(path, f'{pool}.sock') self.reader, self.outbuf, self.sel = wire.Reader(), bytearray(), selectors.DefaultSelector() self.pending = {} # request id: (client socket, verb, deadline) self.waiting = {} # client socket: (bytes so far, deadline) self.next_id, self.pings = 1, 0 + self.accept_error = None def say(self, msg): print(f'fork-fleet-desk serve {self.pool}: {msg}', file=self.log, flush=True) @@ -109,11 +146,19 @@ def run(self): listener.bind(self.sock_path) finally: os.umask(old) + try: # 0660 in the socket group, before anyone may connect (listen) + os.chown(self.sock_path, -1, self.gid) + except PermissionError: + raise Refused(f'cannot give the socket group {self.gid}: this user is not in it') from None + os.chmod(self.sock_path, 0o660) listener.listen(8) listener.setblocking(False) self.sel.register(listener, selectors.EVENT_READ, 'listen') self.say('session open') return self.loop(listener) + except wire.VersionError as e: + self.say(f'session ends: {e}') + return wire.WRONG_VERSION except (wire.WireError, Refused, OSError) as e: self.say(f'session ends: {e}') return 1 @@ -186,7 +231,7 @@ def loop(self, listener): if t > deadline: self.drop(client) for ident, (_, verb, deadline) in list(self.pending.items()): - if t > deadline: + if t > deadline: # the agent should have killed it and answered by now self.reply(ident, wire.answer(wire.UNKNOWN, stderr=f'fork-fleet-desk: pool {self.pool}: {verb}: ' 'no answer in time: it may or may not have run\n')) @@ -202,24 +247,48 @@ def from_agent(self, frame): raise wire.WireError(f'an answer to request {ident}, never sent') def accept(self, listener): + """One socket client, if it may (peer_ok). An error (too many open files, a client gone) costs that client + only, never the session; it is said once until it changes.""" try: client, _ = listener.accept() except BlockingIOError: return - uid = struct.unpack('3i', client.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12))[1] - if uid not in (0, os.geteuid()): + except OSError as e: + if str(e) != self.accept_error: + self.accept_error = str(e) + self.say(f'a socket client could not be taken: {e}') + time.sleep(0.05) # the listener stays readable: do not spin + return + try: + _, uid, gid = struct.unpack('3i', client.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) + except OSError: + client.close() + return + if not self.peer_ok(uid, gid): client.close() return client.setblocking(False) self.waiting[client] = (b'', self.now() + CLIENT) self.sel.register(client, selectors.EVENT_READ, 'client') + def peer_ok(self, uid, gid): + """Root, this user, or a member of the socket group (the dispatcher's user, when it is not root).""" + if uid in (0, os.geteuid()) or gid == self.gid: + return True + try: + return pwd.getpwuid(uid).pw_name in grp.getgrgid(self.gid).gr_mem + except KeyError: + return False + def from_client(self, client): buf, deadline = self.waiting[client] try: data = client.recv(65536) except BlockingIOError: return + except OSError: + self.drop(client) + return buf += data if b'\n' not in buf: if data and len(buf) < wire.MAX_FRAME: @@ -238,7 +307,7 @@ def from_client(self, client): self.answer_client(client, wire.answer(2, stderr=f'fork-fleet-desk: refused: {e}\n')) return ident, self.next_id = self.next_id, self.next_id + 1 - self.pending[ident] = (client, frame['verb'], self.now() + wire.seconds(frame['verb'])) + self.pending[ident] = (client, frame['verb'], self.now() + wire.serve_seconds(frame['verb'])) self.send({**frame, 'id': ident}) def reply(self, ident, frame): @@ -262,8 +331,10 @@ def drop(self, client): # ---------------------------------------------------------------- launch -def launch(pool, verb, args, stdin, path, out=sys.stdout, err=sys.stderr, wait=None): - """The exit code; the desktop's stdout and stderr written to out and err.""" +def launch(pool, verb, args, stdin, path, out=sys.stdout, err=sys.stderr, wait=None, env=os.environ): + """The exit code; the desktop's stdout and stderr written to out and err. Done in `wait` (default + wire.seconds(verb)) all told: connecting, sending and the answer.""" + began = time.monotonic() pool_ok(pool) try: frame = wire.request(verb, args, stdin if verb == 'start' else None) @@ -271,26 +342,31 @@ def launch(pool, verb, args, stdin, path, out=sys.stdout, err=sys.stderr, wait=N raise Refused(f'refused: {e}', 2) from None no_session = Refused(f'pool {pool}: no live desk session (the desktop is off, asleep or not linked): ' f'{verb} was not sent', wire.NO_SESSION) + unknown = Refused(f'pool {pool}: {verb}: no answer from the desktop: it may or may not have run', wire.UNKNOWN) + deadline = began + (wire.seconds(verb) if wait is None else wait) sock_path = os.path.join(path, f'{pool}.sock') + owners = tuple(u for u in (0, os.geteuid(), desk_uid(env)) if u is not None) try: - owner = safe_dir(path, (0, os.geteuid())) + owner = safe_dir(path, owners).st_uid st = os.lstat(sock_path) except FileNotFoundError: raise no_session from None if not stat.S_ISSOCK(st.st_mode) or st.st_uid != owner: raise Refused(f'{sock_path} is not a socket of uid {owner}') with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s: - s.settimeout(CLIENT) + s.settimeout(max(0.01, min(CLIENT, deadline - time.monotonic()))) try: s.connect(sock_path) except (ConnectionRefusedError, FileNotFoundError): raise no_session from None + except socket.timeout: + raise Refused(f'pool {pool}: the desk session takes no connection (busy or stuck): {verb} was not sent', + wire.NO_SESSION) from None except OSError as e: raise Refused(f'cannot reach {sock_path}: {e.strerror}') from None - unknown = Refused(f'pool {pool}: {verb}: no answer from the desktop: it may or may not have run', wire.UNKNOWN) try: + s.settimeout(max(0.01, min(CLIENT, deadline - time.monotonic()))) s.sendall(wire.encode(frame)) - deadline = time.monotonic() + (wire.seconds(verb) if wait is None else wait) buf = b'' while b'\n' not in buf: left = deadline - time.monotonic() @@ -306,6 +382,9 @@ def launch(pool, verb, args, stdin, path, out=sys.stdout, err=sys.stderr, wait=N raise unknown from None out.write(got['stdout']) err.write(got['stderr']) + if got['exit'] == wire.NO_SESSION: # 3 means "nothing was sent" only when the link says it + err.write(f'fork-fleet-desk: pool {pool}: {verb}: the desktop\'s launcher exited {wire.NO_SESSION}\n') + return 1 return got['exit'] @@ -317,8 +396,10 @@ def main(argv=None, env=os.environ): signal.signal(sig, lambda *_: sys.exit(1)) # through serve's finally: the socket goes, clients learn return serve(argv[1], sock_dir(env)) if len(argv) >= 3 and argv[0] == 'launch': + for f in (sys.stdin, sys.stdout, sys.stderr): # odd bytes are refused or shown, never a traceback + f.reconfigure(errors='replace') stdin = sys.stdin.read(wire.MAX_FRAME) if argv[2] == 'start' else None - return launch(argv[1], argv[2], argv[3:], stdin, sock_dir(env)) + return launch(argv[1], argv[2], argv[3:], stdin, sock_dir(env), env=env) raise Refused('usage: serve POOL | launch POOL VERB ARGS...', 2) except Refused as e: print(f'fork-fleet-desk: {e}', file=sys.stderr) diff --git a/.github/scripts/fork-fleet-desk.test.py b/.github/scripts/fork-fleet-desk.test.py index dee31ea0de964..6a3a7a5fed471 100644 --- a/.github/scripts/fork-fleet-desk.test.py +++ b/.github/scripts/fork-fleet-desk.test.py @@ -2,7 +2,9 @@ """Tests of fork-fleet-desk.py (the desk link's always-on side) and of fork-fleet-desk-wire.py, its wire format, with the contract between the wire and fork-fleet-launch.sh and fork-fleet.py. A fake desk agent on pipes, real Unix sockets in a temp directory; no root, no ssh, no network. Run: python3 . Linux only.""" -import importlib.util, inspect, io, os, re, signal, subprocess, sys, tempfile, threading, time, unittest +import importlib.util, inspect, io, json, os, pwd, re, signal, socket, subprocess, sys, tempfile, threading, time, types +import unittest +from unittest import mock sys.dont_write_bytecode = True HERE = os.path.dirname(os.path.abspath(__file__)) @@ -92,7 +94,7 @@ def answer(f): self.assertEqual(self.launch('active'), (0, 'fleet-desk-1\n', '')) self.assertEqual(self.launch('start', 'fleet-desk-5', stdin=START_IN)[0], 0) self.assertEqual(self.launch('fits', '4096'), (1, '', 'not enough memory?[2J\n')) # escapes made harmless - self.assertEqual(oct(os.stat(self.sock).st_mode & 0o777), '0o600') + self.assertEqual(oct(os.stat(self.sock).st_mode & 0o777), '0o660') self.assertEqual(oct(os.stat(self.dir).st_mode & 0o777), '0o700') agent.hang_up() t.join(5) @@ -104,7 +106,6 @@ def test_no_session_fails_at_once_and_sends_nothing(self): self.assertEqual(self.launch('active')[0], wire.NO_SESSION) # no directory at all (after a boot) os.mkdir(self.dir, 0o700) self.assertEqual(self.launch('active')[0], wire.NO_SESSION) # no socket - import socket with socket.socket(socket.AF_UNIX) as s: # a socket left by a killed session s.bind(self.sock) code, _, err = self.launch('active') @@ -240,6 +241,230 @@ def test_serve_cli_cleans_up_on_sigterm(self): self.assertFalse(os.path.exists(self.sock)) self.assertEqual(self.launch('active')[0], wire.NO_SESSION) + # -- reviewer tests (PR #192, correctness lens): each fails when its guard is removed + def test_peer_of_another_uid_is_dropped_without_a_second_user(self): + # SO_PEERCRED faked: the peer looks like uid 4242, in no group of the socket. No second user, no root needed. + agent = Agent(lambda f: wire.answer(0)) + t, _, _ = self.serve(agent) + real = desk.struct + fake = types.SimpleNamespace(unpack=lambda fmt, b: (real.unpack(fmt, b)[0], 4242, 4242)) + with mock.patch.object(desk, 'struct', fake): + code = self.launch('active', wait=2)[0] + self.assertNotEqual(code, 0) + self.assertEqual([f for f in agent.got if 'id' in f], []) # nothing reached the agent + self.assertEqual(self.launch('active')[0], 0) # own uid still served + agent.hang_up(); t.join(5) + + def test_stale_socket_of_a_killed_serve_is_replaced(self): + os.mkdir(self.dir, 0o700) + with socket.socket(socket.AF_UNIX) as s: # left by a SIGKILLed serve + s.bind(self.sock) + agent = Agent(lambda f: wire.answer(0, 'x\n')) + t, result, log = self.serve(agent) + time.sleep(0.2) + self.assertTrue(t.is_alive(), log.getvalue()) + self.assertEqual(self.launch('active')[:2], (0, 'x\n')) + agent.hang_up(); t.join(5) + + def test_serve_rechecks_a_raw_request(self): + agent = Agent(lambda f: wire.answer(0)) + t, _, _ = self.serve(agent) + for raw in (b'{"v":1,"verb":"rm","args":["-rf"]}\n', + b'{"v":1,"verb":"start","args":["x-1"],"stdin":"EVIL=1\\n"}\n', + b'{"v":1,"verb":"active","args":[]}\n{"v":1,"verb":"prune","args":[]}\n'): + with socket.socket(socket.AF_UNIX) as s: + s.connect(self.sock); s.sendall(raw); s.settimeout(3) + got = wire.decode(s.recv(65536).split(b'\n')[0]) + self.assertEqual(got['exit'], 2, raw) + self.assertEqual([f for f in agent.got if 'id' in f], []) + agent.hang_up(); t.join(5) + + def test_serve_own_deadline_answers_unknown(self): + agent = Agent(lambda f: None) + t, _, _ = self.serve(agent) + with mock.patch.object(desk.wire, 'serve_seconds', lambda verb: 0.3): + code, _, err = self.launch('active', wait=5) + self.assertEqual(code, wire.UNKNOWN) + self.assertIn('no answer in time', err) + agent.hang_up(); t.join(5) + + def test_answer_to_an_id_never_sent_ends_the_session(self): + agent = Agent(lambda f: wire.answer(0)) + t, result, log = self.serve(agent) + os.write(agent.to_serve_w, wire.encode({**wire.answer(0), 'id': 99})) + t.join(5) + self.assertEqual(result.get('code'), 1) + self.assertIn('never sent', log.getvalue()) + + def test_launch_refuses_a_socket_of_another_owner(self): + os.mkdir(self.dir, 0o700) + with socket.socket(socket.AF_UNIX) as s: + s.bind(self.sock) + real = os.lstat + def fake(p, *a, **k): + st = real(p, *a, **k) + return types.SimpleNamespace(st_mode=st.st_mode, st_uid=4242) if p == self.sock else st + with mock.patch.object(desk.os, 'lstat', fake): + code, _, err = self.launch('active') + self.assertEqual(code, 1, err) + + def test_silent_client_is_dropped(self): + agent = Agent(lambda f: wire.answer(0)) + t, _, _ = self.serve(agent) + with mock.patch.object(desk, 'CLIENT', 0.3): + s = socket.socket(socket.AF_UNIX); s.connect(self.sock); s.settimeout(5) + self.assertEqual(s.recv(10), b'') # closed after CLIENT s, not held + s.close() + agent.hang_up(); t.join(5) + + # -- fix round 1 (PR #192) + def test_serve_never_runs_as_root(self): + with mock.patch.object(desk.os, 'geteuid', lambda: 0): + with self.assertRaises(desk.Refused) as e: + desk.serve(POOL, self.dir, *os.pipe(), log=io.StringIO()) + self.assertIn('never runs as root', str(e.exception)) + self.assertFalse(os.path.exists(self.dir)) # refused before it touched anything + + def test_root_launch_takes_the_desk_users_directory(self): + # The dispatcher (root) reaches a directory owned by the desk-link user ($FORK_FLEET_DESK_USER, default + # fleet-desk), and no other user's. + os.mkdir(self.dir, 0o710) + me = pwd.getpwuid(os.geteuid()).pw_name + with mock.patch.object(desk.os, 'geteuid', lambda: 0): + for env, want in (({'FORK_FLEET_DESK_USER': me}, wire.NO_SESSION), + ({'FORK_FLEET_DESK_USER': 'no-such-x'}, 1)): + try: + code = desk.launch(POOL, 'active', [], None, self.dir, io.StringIO(), io.StringIO(), env=env) + except desk.Refused as e: + code = e.code + self.assertEqual(code, want, env) + + def test_socket_is_0660_in_the_directory_group_and_the_group_is_let_in(self): + mine = [g for g in os.getgroups() if g != os.getegid()] + gid = mine[0] if mine else os.getegid() # another group of this user, when it has one + os.mkdir(self.dir, 0o710) + os.chown(self.dir, -1, gid) + agent = Agent(lambda f: wire.answer(0, 'ok\n')) + born, chown = [], os.chown # its mode as bound, before chown/chmod + def seen(p, u, g): + born.append(os.stat(p).st_mode & 0o777) + chown(p, u, g) + with mock.patch.object(desk.os, 'chown', seen): + t, _, log = self.serve(agent) + for _ in range(200): + if born or not t.is_alive(): + break + time.sleep(0.01) + self.assertEqual(born, [0o600], log.getvalue()) # owner only until the group is set + st = os.stat(self.sock) + self.assertEqual((oct(st.st_mode & 0o777), st.st_gid), ('0o660', gid), log.getvalue()) + real = desk.struct # the dispatcher's user: another uid, the group + fake = types.SimpleNamespace(unpack=lambda fmt, b: (real.unpack(fmt, b)[0], 4242, gid)) + with mock.patch.object(desk, 'struct', fake): + self.assertEqual(self.launch('active')[:2], (0, 'ok\n')) + agent.hang_up(); t.join(5) + + @unittest.skipIf(os.geteuid() == 0, 'root opens any file') + def test_a_lock_or_directory_left_by_an_older_root_serve_is_refused_clearly(self): + os.mkdir(self.dir, 0o700) + os.close(os.open(os.path.join(self.dir, f'{POOL}.lock'), os.O_CREAT | os.O_WRONLY, 0o000)) + with self.assertRaises(desk.Refused) as e: + desk.serve(POOL, self.dir, *os.pipe(), log=io.StringIO()) + self.assertIn('older serve that ran as root', str(e.exception)) + shut = os.path.join(self.tmp.name, 'shut') # a /run it cannot write: tmpfiles' job + os.mkdir(shut, 0o500) + with self.assertRaises(desk.Refused) as e: + desk.serve(POOL, os.path.join(shut, 'run'), *os.pipe(), log=io.StringIO()) + self.assertIn('systemd-tmpfiles', str(e.exception)) + os.chmod(shut, 0o700) + + def test_wrong_wire_version_says_which_side_to_upgrade(self): + for v, side in ((0, 'upgrade the desktop agent'), (2, "upgrade this host's fork-fleet files")): + r, w = os.pipe() + out_r, out_w = os.pipe() + os.write(w, f'{{"v":{v},"hello":"{POOL}"}}\n'.encode()) + log = io.StringIO() + self.assertEqual(desk.serve(POOL, self.dir, r, out_w, log=log), wire.WRONG_VERSION) + self.assertIn(side, log.getvalue()) + for fd in (r, w, out_r, out_w): + os.close(fd) + + def test_a_launchers_own_exit_3_is_not_no_session(self): + agent = Agent(lambda f: wire.answer(3, '', 'three\n')) + t, _, _ = self.serve(agent) + code, _, err = self.launch('stop', 'fleet-desk-5') + self.assertEqual(code, 1) + self.assertIn("the desktop's launcher exited 3", err) + agent.hang_up(); t.join(5) + + def test_launch_takes_its_budget_all_told(self): + # A stuck serve: connect and send each take nearly CLIENT s, the answer never comes. launch still ends within + # its budget, not CLIENT + CLIENT + budget (start's worst case was 120 s, the dispatcher's own limit). + os.mkdir(self.dir, 0o700) + with socket.socket(socket.AF_UNIX) as s: + s.bind(self.sock) + + class Stuck: + def __init__(self, *a): + self.t = None + def settimeout(self, t): + self.t = t + def connect(self, _): + time.sleep(max(0, self.t - 0.05)) + def sendall(self, _): + time.sleep(max(0, self.t - 0.05)) + def recv(self, _): + time.sleep(self.t) + raise socket.timeout() + def __enter__(self): + return self + def __exit__(self, *a): + pass + began = time.monotonic() + with mock.patch.object(desk, 'CLIENT', 1), mock.patch.object(desk.socket, 'socket', Stuck): + code = self.launch('active', wait=1.5)[0] + self.assertEqual(code, wire.UNKNOWN) + self.assertLess(time.monotonic() - began, 1.5 + 0.4) + + def test_an_accept_error_costs_one_client_not_the_session(self): + agent = Agent(lambda f: wire.answer(0, 'ok\n')) + t, result, log = self.serve(agent) + real, failed = socket.socket.accept, [] + + def accept(sock): + if not failed: + failed.append(1) + raise OSError(24, 'Too many open files') + return real(sock) + with mock.patch.object(socket.socket, 'accept', accept): + first = self.launch('active', wait=3)[0] + self.assertEqual(failed, [1]) + self.assertEqual(first, 0) # it waited in the backlog, then was taken + self.assertTrue(t.is_alive(), log.getvalue()) # the session goes on + self.assertEqual(self.launch('active')[:2], (0, 'ok\n')) + self.assertIn('Too many open files', log.getvalue()) + agent.hang_up(); t.join(5) + + def test_odd_names_and_bytes_never_crash_launch(self): + env = {'PATH': os.environ['PATH'], 'FORK_FLEET_DESK_DIR': self.dir} + for argv, stdin in (([b'launch', b'\xff\n', b'active'], b''), ([b'launch', b'desk', b'stop', b'\xff-1'], b''), + ([b'launch', b'desk', b'\xc3\xa9'], b''), + ([b'launch', b'desk', b'start', b'x-1'], b'\xff\n'), + ([b'launch', b'de"sk', b'fits', b'1\x00'.replace(b'\x00', b'\x01')], b'')): + got = subprocess.run([sys.executable, '-I', DESK.encode(), *argv], env=env, input=stdin, + capture_output=True, timeout=30) + self.assertEqual(got.returncode, 2, (argv, got.stderr)) + self.assertNotIn(b'Traceback', got.stderr) + strict = [io.TextIOWrapper(io.BytesIO(b), encoding='utf-8', errors='strict') for b in (b'\xff\n', b'', b'')] + with mock.patch.multiple(desk.sys, stdin=strict[0], stdout=strict[1], stderr=strict[2]): # a UTF-8 locale + self.assertEqual(desk.main(['launch', POOL, 'start', 'x-1'], env), 2) + agent = Agent(lambda f: wire.answer(0, 'café \ud800\n')) # odd text back: shown, not a crash + t, _, _ = self.serve(agent) + got = subprocess.run([sys.executable, '-I', DESK, 'launch', POOL, 'active'], env=env, capture_output=True, + timeout=30) + self.assertEqual((got.returncode, got.stdout.decode()), (0, 'café ?\n'), got.stderr) + agent.hang_up(); t.join(5) + class Wire(unittest.TestCase): """The wire's rules, and its contract with fork-fleet-launch.sh and fork-fleet.py (one source each).""" @@ -265,8 +490,16 @@ def test_patterns_are_the_launchers_and_the_dispatchers(self): self.assertIn(f'$url =~ ^{wire.URL.pattern}$ ]]', self.LAUNCH) def test_every_dispatcher_verb_answers_before_the_dispatcher_gives_up(self): + # The agent kills a verb at its limit, serve answers 75 after that, launch ends after that, and all of it + # before Host._do's timeout. The verbs are read from Host itself, so a new one is checked too. limit = int(re.search(r'timeout=([0-9]+)', inspect.getsource(ff.Host._do)).group(1)) - for verb in ('start', 'ready', 'active', 'fits', 'stop', 'prune'): + src = ''.join(inspect.getsource(f) for f in vars(ff.Host).values() if inspect.isfunction(f)) + verbs = set(re.findall(r"self\._(?:must|do)\('([a-z-]+)'", src)) + self.assertEqual(verbs, {'start', 'ready', 'active', 'fits', 'stop', 'prune'}) + for verb in wire.VERBS: + self.assertLess(wire.limit(verb), wire.serve_seconds(verb), verb) + self.assertLess(wire.serve_seconds(verb), wire.seconds(verb), verb) + for verb in verbs: self.assertLess(wire.seconds(verb), limit, verb) def test_the_dispatchers_start_input_passes(self): @@ -276,6 +509,37 @@ def test_the_dispatchers_start_input_passes(self): with self.assertRaises(wire.WireError): wire.check_start_input(bad) + def test_deep_json_is_a_wire_error(self): + line = b'[' * 100000 + b']' * 100000 # under MAX_FRAME, too deep for json + self.assertLess(len(line), wire.MAX_FRAME) + with self.assertRaises(wire.WireError): + wire.decode(line) + with self.assertRaises(wire.WireError): + wire.Reader().feed(line + b'\n') + + def test_answer_text_is_clean_of_lone_surrogates(self): + got = wire.check_answer(json.loads('{"v":1,"exit":0,"stdout":"a\\ud800b","stderr":""}'), with_id=False) + self.assertEqual(got['stdout'], 'a?b') + got['stdout'].encode('utf-8') + + def test_text_is_sized_in_bytes(self): + frame = wire.answer(0, '\u00e9' * wire.MAX_TEXT, 'x' * (wire.MAX_TEXT + 5)) # 6 bytes each on the wire + self.assertLessEqual(len(wire.encode(frame)), wire.MAX_FRAME) + for key in ('stdout', 'stderr'): + self.assertLessEqual(wire.text_size(frame[key]), wire.MAX_TEXT) + self.assertTrue(frame[key].endswith(wire.CUT)) + self.assertEqual(wire.fit('short \u00e9'), 'short \u00e9') + over = {'v': 1, 'exit': 0, 'stdout': '\u00e9' * (wire.MAX_TEXT // 6 + 1), 'stderr': ''} # few characters + with self.assertRaises(wire.WireError): + wire.check_answer(over, with_id=False) + + def test_start_input_ends_in_a_newline(self): + with self.assertRaisesRegex(wire.WireError, 'newline'): + wire.check_start_input(START_IN[:-1]) + with self.assertRaisesRegex(wire.WireError, 'newline'): + wire.check_start_input(START_IN[:-1] + 'x') + wire.check_start_input(START_IN) + def test_frames(self): r = wire.Reader() frame = wire.request('start', ['x-1'], START_IN, ident=4) @@ -294,5 +558,27 @@ def test_frames(self): wire.check_request({'v': 1, 'verb': 'active', 'args': [], 'stdin': 'x'}, with_id=False) +class Install(unittest.TestCase): + """The runbook and the key tool (fork-fleet-desk-key.py, row D4) name this program the same way.""" + DOC = open(os.path.join(HERE, '..', 'fork-fleet.md'), encoding='utf-8').read() + + def test_the_runbook_installs_and_names_the_desk_link(self): + self.assertIn(f'command="{desk.FORCED} POOL"', self.DOC) + self.assertEqual(desk.FORCED.split()[2], desk.INSTALLED) + self.assertIn(f"echo '{desk.TMPFILES}'", self.DOC) + for f in ('fork-fleet-desk.py', 'fork-fleet-desk-wire.py'): + self.assertIn(f'scripts/{f}:{f}:644', self.DOC) + self.assertIn(f'--shell /bin/sh {desk.USER}', self.DOC) # nologin would close the link + + def test_the_key_tool_writes_this_forced_command(self): + tool = os.path.join(HERE, 'fork-fleet-desk-key.py') + if not os.path.exists(tool): + self.skipTest('fork-fleet-desk-key.py (row D4) is not here yet') + src = open(tool, encoding='utf-8').read() + self.assertIn(f"DESK = '{desk.INSTALLED}'", src) + self.assertIn("SERVE = f'python3 -I {DESK} serve'", src) + self.assertIn(f"USER = '{desk.USER}'", src) + + if __name__ == '__main__': unittest.main() diff --git a/.github/scripts/fork-fleet.py b/.github/scripts/fork-fleet.py index 8ab5ca641e5b5..1bc53fd8882f8 100644 --- a/.github/scripts/fork-fleet.py +++ b/.github/scripts/fork-fleet.py @@ -908,7 +908,7 @@ def new_state(): """Kept between passes only to pace heartbeats, idle checks, retries, messages and the newest runner release. GitHub (runs, jobs, runner registrations) and the host (units) hold everything else, so a restarted dispatcher picks up where one stopped.""" - return {'beacon': {}, 'idle': {}, 'tries': {}, 'said': set(), 'newest': None} + return {'beacon': {}, 'idle': {}, 'tries': {}, 'said': set(), 'newest': None, 'down': {}} RELEASE_EVERY = 3600 # seconds between reads of the newest actions/runner release @@ -933,6 +933,24 @@ def newest_runner(gh, state, now): return state['newest'][0] +DOWN_EVERY = 600 # seconds between "still left alone" lines of a pool that stays down for the same reason + + +def say_down(state, pool, why, t): + """A pool's host failed: said at once; while it keeps failing for the same reason (a desktop asleep all night), + again only every DOWN_EVERY s, with the count of passes.""" + downs = state.setdefault('down', {}) + last = downs.get(pool) + if last and last['why'] == why: + last['passes'] += 1 + if t - last['said'] >= DOWN_EVERY: + last['said'] = t + say(f'::warning::fork-fleet: {why}; pool {pool} is still left alone ({last["passes"]} passes in a row)') + return + downs[pool] = {'why': why, 'passes': 1, 'said': t} + say(f'::warning::fork-fleet: {why}; pool {pool} is left alone for this pass, the other pools go on') + + def once(state, key, msg): if key not in state['said']: state['said'].add(key) @@ -952,7 +970,7 @@ def host(p, verb, *args): return getattr(hosts[p['name']], verb)(*args) except HostError as e: down[p['name']] = str(e) - say(f'::warning::fork-fleet: {e}; pool {p["name"]} is left alone for this pass, the other pools go on') + say_down(state, p['name'], str(e), now()) return None runs = {} @@ -1092,6 +1110,10 @@ def host(p, verb, *args): continue set_variable(gh, p['beacon'], json.dumps(beat, separators=(',', ':'))) state['beacon'][p['name']] = ((free, why), t) + for p in pools: + gone = p['name'] not in down and state.setdefault('down', {}).pop(p['name'], None) + if gone and gone['passes'] > 1: + say(f'fork-fleet: pool {p["name"]} answers again, after {gone["passes"]} passes left alone') return 0 diff --git a/.github/scripts/fork-fleet.test.py b/.github/scripts/fork-fleet.test.py index 917abd3686c48..2f0a11b03dd1a 100644 --- a/.github/scripts/fork-fleet.test.py +++ b/.github/scripts/fork-fleet.test.py @@ -1131,6 +1131,23 @@ def test_a_host_that_cannot_answer_skips_only_its_pool(self): self.assertNotIn(self.name(40, FIRST), [r['name'] for r in self.gh.runners]) self.assertNotIn('why', json.loads(self.gh.variables[POOLS[FIRST]['beacon']])) + def test_a_pool_down_for_the_same_reason_is_said_once_then_every_ten_minutes(self): + """A desktop asleep all night: its pool's host fails every pass (20 s); the log gets a line, then one every + DOWN_EVERY s with the count, then one when it answers again; not one per pass.""" + self.fake[FIRST].fail = 'active' + for _ in range(5): + self.dispatch() + self.clock[0] += 20 + self.assertEqual(self.log.count(f'pool {FIRST} is left alone'), 1) + self.assertNotIn('still left alone', self.log) + self.clock[0] += ff.DOWN_EVERY + self.assertIn(f'pool {FIRST} is still left alone (6 passes in a row)', self.dispatch()) + self.assertNotIn('still left alone', self.dispatch()) + self.fake[FIRST].fail = None + self.assertIn(f'pool {FIRST} answers again, after 7 passes left alone', self.dispatch()) + self.fake[FIRST].fail = 'active' + self.assertIn(f'pool {FIRST} is left alone for this pass', self.dispatch()) # a new outage: said at once + def test_a_failed_start_skips_only_its_pool_and_is_tried_again(self): self.gh.add(a_run(), a_job(11, job_labels(WAITING, 'lint')), a_job(12, job_labels(WAITING, 'rust'))) self.gh.add(a_run(6), a_job(13, job_labels(FIRST, 'build')))