diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 4d97ac429c426..d7815e726cf73 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -62,6 +62,7 @@ fork-kit/heavy/tests/install.test.py fork-kit/heavy/tests/sweep.test.py # A real systemd user manager (the step starts one for the runner user). fork-kit/heavy/tests/heavy.test.py run by .github/workflows/fork-tests.yml +fork-kit/pc-agents/tests/check.test.py # Upstream's tests, run where fork-tests.yml follows upstream's ci.yml. .github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 9b4d31a5a5991..696b307ef5188 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -1,9 +1,9 @@ -// Fork only: every `*.test.*` under .github/scripts, fork-ci and fork-kit must be listed in .github/fork-ci-tests.txt. -// A plain entry runs here by its extension (RUNNER); ` run by ` is run by that file, -// which must name the test and cannot be a (disabled) upstream workflow. Unlisted or runner-less -// tests fail; a listed test that is gone fails once the tested history had it, a note before. +// Fork only: every `*.test.*` under .github/scripts, fork-ci and fork-kit is listed in .github/fork-ci-tests.txt and +// runs here by extension (RUNNER), FORK_CI_TEST_JOBS at once (default: CPU count), or `run by ` (not an upstream +// workflow; it names the test). Unlisted or runner-less tests fail; a listed one that is gone fails once history had it. const childProcess = require("node:child_process"); const fs = require("node:fs"); +const os = require("node:os"); const path = require("node:path"); const ROOT = path.resolve(__dirname, "../.."); @@ -119,24 +119,74 @@ function readFile(file) { } } -function main() { +// Calls start(item) for every item, at most `jobs` at once, each as soon as a slot is free. +// start returns a promise; the results come back in the items' order. +async function runAll(items, jobs, start) { + const results = new Array(items.length); + let next = 0; + async function worker() { + while (next < items.length) { + const index = next++; + results[index] = await start(items[index]); + } + } + await Promise.all(Array.from({ length: Math.max(1, Math.min(jobs, items.length)) }, worker)); + return results; +} + +function jobCount(env = process.env) { + const jobs = Number(env.FORK_CI_TEST_JOBS); + return Number.isInteger(jobs) && jobs > 0 ? jobs : os.availableParallelism(); +} + +// Runs one test file, its stdout and stderr kept together, and prints them as one group when it ends. +function runTest(file) { + const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]]; + const started = Date.now(); + return new Promise((resolve) => { + const child = childProcess.spawn(command, [...args, file], { + cwd: ROOT, + stdio: ["ignore", "pipe", "pipe"], + }); + const output = []; + child.stdout.on("data", (chunk) => output.push(chunk)); + child.stderr.on("data", (chunk) => output.push(chunk)); + let ended = false; + const done = (status, signal) => { + if (ended) return; + ended = true; + // Stop reading pipes a left-behind process still holds, so they do not keep this script alive. + child.stdout.destroy(); + child.stderr.destroy(); + const seconds = ((Date.now() - started) / 1000).toFixed(1); + process.stdout.write(`::group::${file} (${seconds} s)\n`); + process.stdout.write(Buffer.concat(output)); + process.stdout.write("\n::endgroup::\n"); + if (status !== 0) + console.log(`::error file=${file}::${file} failed (exit ${status ?? signal})`); + resolve({ file, status, seconds }); + }; + child.on("error", (error) => { + output.push(Buffer.from(`cannot start ${command}: ${error.message}\n`)); + done(null, "spawn error"); + }); + child.on("close", done); + // A process the test left behind may hold the pipes open: the test is done when it exits. + child.on("exit", (status, signal) => setTimeout(() => done(status, signal), 5000).unref()); + }); +} + +async function main() { const entries = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8")); const result = plan(entries, presentTests(), everInHistory, readFile); const rows = []; let failed = result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length; - for (const file of result.run) { - const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]]; - const started = Date.now(); - console.log(`::group::${file}`); - const run = childProcess.spawnSync(command, [...args, file], { cwd: ROOT, stdio: "inherit" }); - console.log("::endgroup::"); - const seconds = ((Date.now() - started) / 1000).toFixed(1); - if (run.status !== 0) { - failed += 1; - console.log(`::error file=${file}::${file} failed (exit ${run.status ?? run.signal})`); - } - rows.push(`| \`${file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${seconds} s |`); + for (const run of await runAll(result.run, jobCount(), runTest)) { + if (run.status !== 0) failed += 1; + rows.push( + `| \`${run.file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${run.seconds} s |`, + ); } for (const file of result.unlisted) console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`); @@ -175,6 +225,10 @@ function main() { if (failed) process.exitCode = 1; } -module.exports = { plan, readList, findTests, presentTests }; +module.exports = { plan, readList, findTests, presentTests, runAll, jobCount }; -if (require.main === module) main(); +if (require.main === module) + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index da0290dc92daa..3fbda8b020161 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -5,7 +5,14 @@ const os = require("node:os"); const path = require("node:path"); const test = require("node:test"); -const { plan, readList, findTests, presentTests } = require("./fork-ci-tests.cjs"); +const { + plan, + readList, + findTests, + presentTests, + runAll, + jobCount, +} = require("./fork-ci-tests.cjs"); const never = () => false; const here = (...files) => files.map((file) => ({ file, runBy: null })); @@ -143,3 +150,174 @@ test("an unlisted test under .github/scripts or fork-ci fails the run", () => { fs.rmSync(root, { recursive: true, force: true }); } }); + +test("runAll runs at most `jobs` at once, starts the next as soon as a slot frees, keeps the order", async () => { + const runs = new Map(); // item -> resolve of its run, in start order + const waiting = []; // [count, resolve]: resolved once that many runs started + const started = (count) => + new Promise((resolve) => (runs.size >= count ? resolve() : waiting.push([count, resolve]))); + let running = 0; + let most = 0; + const all = runAll(["a", "b", "c", "d"], 2, (item) => { + running += 1; + most = Math.max(most, running); + const done = new Promise((resolve) => runs.set(item, resolve)).then(() => { + running -= 1; + return item.toUpperCase(); + }); + for (const [count, resolve] of waiting) if (runs.size >= count) resolve(); + return done; + }); + await started(2); + assert.deepEqual([...runs.keys()], ["a", "b"]); // two at once, c waits for a free slot + runs.get("b")(); // the second ends first: its slot takes c + await started(3); + assert.deepEqual([...runs.keys()], ["a", "b", "c"]); + runs.get("a")(); + await started(4); + runs.get("d")(); + runs.get("c")(); + assert.deepEqual(await all, ["A", "B", "C", "D"]); + assert.equal(most, 2); + assert.deepEqual(await runAll([], 4, async () => 1), []); +}); + +test("jobCount reads FORK_CI_TEST_JOBS, else the CPU count", () => { + assert.equal(jobCount({ FORK_CI_TEST_JOBS: "3" }), 3); + for (const bad of [undefined, "", "0", "-2", "1.5", "many"]) + assert.equal(jobCount({ FORK_CI_TEST_JOBS: bad }), os.availableParallelism(), String(bad)); +}); + +// Runs the real script in a scratch tree whose list names `tests` ({ file: shell text }). +function runScript(tests, jobs) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fork-ci-tests-")); + const write = (file, text) => { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + }; + write( + ".github/scripts/fork-ci-tests.cjs", + fs.readFileSync(require.resolve("./fork-ci-tests.cjs")), + ); + write(".github/fork-ci-tests.txt", Object.keys(tests).join("\n") + "\n"); + for (const [file, text] of Object.entries(tests)) write(file, text); + const env = { + ...process.env, + FORK_CI_TEST_JOBS: String(jobs), + LEFT: path.join(root, "left.pid"), + MARKS: root, + }; + delete env.GITHUB_STEP_SUMMARY; + const run = childProcess.spawnSync("node", [".github/scripts/fork-ci-tests.cjs"], { + cwd: root, + env, + encoding: "utf8", + timeout: 60000, + }); + return { root, run }; +} + +// A test file that passes only when `other` starts while it runs (it waits up to 5 s for the other's mark). +const meets = (name, other) => + `echo ${name}-start; : > "$MARKS/${name}"\n` + + `for i in $(seq 50); do [ -e "$MARKS/${other}" ] && { echo ${name}-end; exit 0; }; sleep 0.1; done\n` + + `echo "${other} did not run beside ${name}"; exit 1\n`; + +test("test files run in parallel, each one's output in its own group, and a failure fails the run", () => { + const { root, run } = runScript( + { + "fork-ci/a.test.sh": meets("a", "b"), + "fork-ci/b.test.sh": meets("b", "a"), + "fork-ci/c.test.sh": "echo c-start; echo c-end; exit 3\n", + }, + 3, + ); + try { + assert.equal(run.status, 1, run.stdout + run.stderr); + for (const name of ["a", "b", "c"]) + assert.match( + run.stdout, + new RegExp( + `::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`, + ), + ); + assert.doesNotMatch(run.stdout, /::error file=fork-ci\/[ab]\.test\.sh/); + assert.match( + run.stdout, + /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/, + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("FORK_CI_TEST_JOBS=1 runs one file at a time", () => { + const { root, run } = runScript( + { "fork-ci/a.test.sh": meets("a", "b"), "fork-ci/b.test.sh": meets("b", "a") }, + 1, + ); + try { + assert.equal(run.status, 1, run.stdout + run.stderr); + assert.match(run.stdout, /b did not run beside a/); // b started only after a ended + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("a process a test leaves behind holding its output does not hang the run", () => { + const { root, run } = runScript( + { "fork-ci/left.test.sh": 'sleep 60 & echo $! > "$LEFT"; echo done\n' }, + 1, + ); + try { + assert.equal(run.error, undefined, String(run.error)); // not killed at the 60 s limit: the run ended + assert.equal(run.status, 0, run.stdout + run.stderr); + assert.match(run.stdout, /::group::fork-ci\/left\.test\.sh[^\n]*\ndone\n/); + } finally { + try { + process.kill(Number(fs.readFileSync(path.join(root, "left.pid"), "utf8"))); + } catch {} + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +// A pipe into a reader that stops early (`grep -q`, `grep -m`, `head`) kills the writer with SIGPIPE +// whenever it has more to write; under `pipefail` that reads as a failed check, more often under load. +const earlyExitPipe = + /(? + text + .split("\n") + .map((line, index) => ({ line, number: index + 1 })) + .filter(({ line }) => !/^\s*#/.test(line) && earlyExitPipe.test(line)); + +test("a pipe into grep -q, grep -m or head is found; reading the whole output first is not", () => { + const found = earlyExitPipes( + [ + 'sed -n "/a/,$p" f | grep -q x', + "! calls | grep -vqE y", + "git show c | grep -qx z", + "seq 9 | grep -m1 3", + 'sed p <<< "$s" | head -n 1', + 'grep -q x <<< "$(sed -n p f)"', + "a || grep -q x f", + "# a comment: `x | grep -q` stops early", + '[[ -z "$(grep -E a f | grep -vE b)" ]]', + ].join("\n"), + ).map(({ number }) => number); + assert.deepEqual(found, [1, 2, 3, 4, 5]); +}); + +test("no listed shell test pipes into a reader that stops early", () => { + const root = path.resolve(__dirname, "../.."); + const shellTests = readList(fs.readFileSync(path.join(root, ".github/fork-ci-tests.txt"), "utf8")) + .map(({ file }) => file) + .filter((file) => file.endsWith(".sh") && fs.existsSync(path.join(root, file))); + assert.ok(shellTests.length > 0); + const found = shellTests.flatMap((file) => + earlyExitPipes(fs.readFileSync(path.join(root, file), "utf8")).map( + ({ line, number }) => `${file}:${number}: ${line.trim()}`, + ), + ); + assert.deepEqual(found, []); +}); diff --git a/.github/scripts/fork-follow.test.sh b/.github/scripts/fork-follow.test.sh index 72eafc76779d3..cb5a252047a79 100755 --- a/.github/scripts/fork-follow.test.sh +++ b/.github/scripts/fork-follow.test.sh @@ -441,7 +441,7 @@ plan check "plan: upstream edits a deleted listed file -> the conflict is resolved by deleting it, rebase" \ eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.11 ]] && ! in_candidate ci.yml && [[ ! -e "$root/report.md" ]]' check "plan: the resolved rebase keeps the fork's patches and its list" \ - eval 'action_is rebase && git -C "$root/work" cat-file -e "$(out candidate):feature.txt" && git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt" | grep -qx .github/workflows/ci.yml' + eval 'action_is rebase && git -C "$root/work" cat-file -e "$(out candidate):feature.txt" && grep -qx .github/workflows/ci.yml <<< "$(git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt")"' # Re-run: an earlier run published this candidate, then stopped before the branch moved. The # next run rebuilds the same files and advances to that release instead of building again. released="$(out candidate)" @@ -470,7 +470,7 @@ new_fork v0.1.11 fork_commit "fork: list ci.yml" 'echo .github/workflows/ci.yml >> .github/fork-removed-workflows.txt' plan check "plan: upstream brings a listed file back -> deleted again on the candidate, rebase" \ - eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.12 ]] && ! in_candidate ci.yml && git -C "$root/work" log -1 --format=%s "$(out candidate)" | grep -q "fork: delete upstream workflows listed"' + eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.12 ]] && ! in_candidate ci.yml && grep -q "fork: delete upstream workflows listed" <<< "$(git -C "$root/work" log -1 --format=%s "$(out candidate)")"' new_upstream new_fork v0.1.10 @@ -506,7 +506,7 @@ new_fork v0.1.10 fork_commit "fork: delete ci.yml, CRLF list" "printf '# deleted\\r\\n.github/workflows/ci.yml\\r\\n' > .github/fork-removed-workflows.txt && rm .github/workflows/ci.yml" plan check "plan: a list with CRLF line ends is read like one with LF, the listed file stays deleted" \ - eval 'action_is rebase && ! in_candidate ci.yml && git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt" | grep -q $'"'"'ci.yml\r$'"'"'' + eval 'action_is rebase && ! in_candidate ci.yml && grep -q $'"'"'ci.yml\r$'"'"' <<< "$(git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt")"' # Upstream renames a listed file: git reports a rename/delete conflict on the new name, which the # list does not name, so the plan stops and names it rather than shipping it under that name. @@ -1054,9 +1054,9 @@ check "workflow: advance's if: is a conjunction (no top-level ||) of the plan's check "workflow: the move step runs only when the checks before it passed (no if:)" \ eval 'awk "/- name: Move the series branch/ { s = 1; next } s && /^ - / { exit } s && /^ if:/ { bad = 1 } END { exit bad }" <<< "$advance_job"' check "workflow: action none deletes a leftover candidate branch" \ - eval 'grep -B2 -A3 "action == .none." "$workflow" | grep -q "fork-follow-push.sh cleanup"' + eval 'grep -q "fork-follow-push.sh cleanup" <<< "$(grep -B2 -A3 "action == .none." "$workflow")"' check "workflow: the last sweep reports runs since the run began" \ - eval 'sed -n "/^ report:/,\$p" "$workflow" | grep -q "SINCE: \${{ needs.guard.outputs.started }}"' + eval 'grep -q "SINCE: \${{ needs.guard.outputs.started }}" <<< "$(sed -n "/^ report:/,\$p" "$workflow")"' # Each job that reads FORK_PUSH_TOKEN runs in the Environment that holds it, named as in env. token_jobs="$(grep -vF "$presence" "$workflow" | awk '/^ [a-z_-]+:$/ { job = $1 } /secrets\.FORK_PUSH_TOKEN/ { print job }' | sort -u)" env_jobs="$(awk '/^ [a-z_-]+:$/ { job = $1 } /^ environment: fork-push$/ { print job }' "$workflow" | sort -u)" @@ -1067,9 +1067,9 @@ check "workflow: the jobs that read FORK_PUSH_TOKEN, and only they, run in the E check "workflow: no job uses actions/checkout; four plain-git checkouts, none with a token" \ eval '! grep -q "actions/checkout@" "$workflow" && [[ $(grep -c "^ git init -q$" "$workflow") == 4 ]]' check "workflow: every action is pinned by commit" \ - eval '! grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vqE "@[0-9a-f]{40}( |$)"' + eval '[[ -z "$(grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vE "@[0-9a-f]{40}( |$)")" ]]' check "workflow: the report runs on every run (also action none)" \ - eval 'sed -n "/^ report:/,\$p" "$workflow" | grep -q "^ if: always()$"' + eval 'grep -q "^ if: always()$" <<< "$(sed -n "/^ report:/,\$p" "$workflow")"' check "workflow: upstream workflows are swept after each push" \ eval '[[ $(grep -c "fork-workflows.sh" "$workflow") -ge 3 ]]' diff --git a/.github/scripts/fork-interaction-limit.test.sh b/.github/scripts/fork-interaction-limit.test.sh index 63424f684b844..0c6a499be68c3 100644 --- a/.github/scripts/fork-interaction-limit.test.sh +++ b/.github/scripts/fork-interaction-limit.test.sh @@ -107,8 +107,8 @@ expect() { # expect LABEL 'CONDITION' (shell, evaluated) } rc_is() { [[ "$rc" == "$1" ]]; } says() { grep -qF -- "$1" <<< "$out"; } -called() { calls | grep -qF -- "$1"; } -not_called() { ! calls | grep -qF -- "$1"; } +called() { grep -qF -- "$1" <<< "$(calls)"; } +not_called() { ! grep -qF -- "$1" <<< "$(calls)"; } # --- check ------------------------------------------------------------------------------------- reset; limit collaborators_only 180; run check diff --git a/.github/scripts/fork-run-alert.test.sh b/.github/scripts/fork-run-alert.test.sh index ea5e77c6751dc..ca4311e340760 100644 --- a/.github/scripts/fork-run-alert.test.sh +++ b/.github/scripts/fork-run-alert.test.sh @@ -83,8 +83,8 @@ expect() { # expect LABEL 'CONDITION' (shell, evaluated) } rc_is() { [[ "$rc" == "$1" ]]; } says() { grep -qF -- "$1" <<< "$out"; } -called() { calls | grep -qF -- "$1"; } -not_called() { ! calls | grep -qF -- "$1"; } +called() { grep -qF -- "$1" <<< "$(calls)"; } +not_called() { ! grep -qF -- "$1" <<< "$(calls)"; } issues() { jq length "$root/state/issues.json" 2> /dev/null || echo 0; } reset; run failure --message "The daily secret sweep failed" diff --git a/.github/scripts/fork-secret-scan.test.sh b/.github/scripts/fork-secret-scan.test.sh index 3263f60629f82..adddb5b7a147b 100755 --- a/.github/scripts/fork-secret-scan.test.sh +++ b/.github/scripts/fork-secret-scan.test.sh @@ -65,7 +65,7 @@ elif [[ -n "${GITLEAKS_CONFIG:-}" ]]; then settings="$(cat "$GITLEAKS_CONFIG")" elif [[ -n "${GITLEAKS_CONFIG_TOML:-}" ]]; then settings="$GITLEAKS_CONFIG_TOML" elif [[ -f "$source/.gitleaks.toml" ]]; then settings="$(cat "$source/.gitleaks.toml")" fi -extend_path="$(sed -nE "s/^[[:space:]]*path[[:space:]]*=[[:space:]]*[\"']([^\"']*)[\"'].*/\1/p" <<< "$settings" | head -n 1)" +extend_path="$(sed -nE "s/^[[:space:]]*path[[:space:]]*=[[:space:]]*[\"']([^\"']*)[\"'].*/\1/p" <<< "$settings" | sed -n 1p)" if [[ "$settings" == *"[extend]"* && -n "$extend_path" && ! -f "$extend_path" ]]; then echo "FTL Failed to load config error=\"failed to load extended config, err: open $extend_path: no such file or directory\"" exit 1 diff --git a/apps/server/src/policy/managedSettings.forkKeys.test.ts b/apps/server/src/policy/managedSettings.forkKeys.test.ts new file mode 100644 index 0000000000000..99f818179d0ab --- /dev/null +++ b/apps/server/src/policy/managedSettings.forkKeys.test.ts @@ -0,0 +1,85 @@ +// Fork: a managed policy locks the fork's own settings like upstream's. The pc-agents host kit +// (fork-kit/pc-agents) writes a policy that locks the beta switch, automatic updates and the +// child ceiling (agentSlots, one field per dotted path); this pins that each of them is accepted +// and held, and that a partial object is refused (the kit refuses object values for that reason). + +import { DEFAULT_SERVER_SETTINGS, type ServerSettings } from "@t3tools/contracts"; +import { assert, it } from "@effect/vitest"; + +import { makeManagedSettings } from "./managedSettings.ts"; + +const KIT_POLICY = { + linkedEnvironmentsEnabled: { value: true, locked: true }, + automaticUpdates: { value: false, locked: true }, + "agentSlots.enabled": { value: true, locked: true }, + "agentSlots.maxPerMachine": { value: 2, locked: true }, +}; + +const managed = () => { + const result = makeManagedSettings(KIT_POLICY, "full-access"); + if (result._tag !== "Valid") assert.fail(result.reason); + return result.settings; +}; + +const userSettings: ServerSettings = { + ...DEFAULT_SERVER_SETTINGS, + linkedEnvironmentsEnabled: false, + automaticUpdates: true, + agentSlots: { ...DEFAULT_SERVER_SETTINGS.agentSlots, enabled: false, maxPerMachine: 9 }, +}; + +it("accepts the pc-agents kit's locks on fork settings", () => { + assert.deepEqual([...managed().lockedPaths].toSorted(), [ + "agentSlots.enabled", + "agentSlots.maxPerMachine", + "automaticUpdates", + "linkedEnvironmentsEnabled", + ]); +}); + +it("lays the locked fork values over the user's", () => { + const applied = managed().applyLocks(userSettings); + assert.strictEqual(applied.linkedEnvironmentsEnabled, true); + assert.strictEqual(applied.automaticUpdates, false); + assert.strictEqual(applied.agentSlots.enabled, true); + assert.strictEqual(applied.agentSlots.maxPerMachine, 2); +}); + +it("refuses a write that raises the locked child ceiling or turns the beta off", () => { + const settings = managed(); + const current = settings.applyLocks(userSettings); + const raise = settings.vetWrite( + current, + { ...current, agentSlots: { ...current.agentSlots, maxPerMachine: 5 } }, + { agentSlots: { maxPerMachine: 5 } }, + ); + assert.deepEqual(raise, { _tag: "Locked", path: "agentSlots.maxPerMachine" }); + const off = settings.vetWrite( + current, + { ...current, linkedEnvironmentsEnabled: false }, + { linkedEnvironmentsEnabled: false }, + ); + assert.deepEqual(off, { _tag: "Locked", path: "linkedEnvironmentsEnabled" }); +}); + +it("locks one field of a fork setting and leaves the others to the user", () => { + const result = makeManagedSettings({ "agentSlots.maxPerMachine": { value: 2, locked: true } }); + if (result._tag !== "Valid") assert.fail(result.reason); + assert.strictEqual(result.settings.applyLocks(userSettings).agentSlots.maxPerMachine, 2); + assert.strictEqual(result.settings.applyLocks(userSettings).agentSlots.enabled, false); +}); + +it("takes an object only as a fork setting's complete value", () => { + const partial = makeManagedSettings({ + agentSlots: { value: { maxPerMachine: 2 }, locked: true }, + }); + assert.strictEqual(partial._tag, "Invalid"); + const whole = makeManagedSettings({ + agentSlots: { + value: { ...DEFAULT_SERVER_SETTINGS.agentSlots, maxPerMachine: 2 }, + locked: true, + }, + }); + if (whole._tag !== "Valid") assert.fail(whole.reason); + assert.strictEqual(whole.settings.applyLocks(userSettings).agentSlots.maxPerMachine, 2); +}); diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md new file mode 100644 index 0000000000000..73213ec8991d4 --- /dev/null +++ b/fork-kit/pc-agents/README.md @@ -0,0 +1,146 @@ +# pc-agents: run T3's agents on a Windows PC, walled in a WSL2 distro of their own + +A PC can take agent work from another T3 (a linked environment, with **Full access**). Full access runs as the PC +T3's user, so the link is not the wall: the distro is. This kit makes a fresh WSL2 distro into that wall and runs +T3 in it as `agent`, a user with no sudo, under a root-owned system unit. Everything that holds the wall is root's, +so the agent cannot undo it: + +| Wall | What the kit does | +| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` and `/run/tailscale` (the Tailscale daemon's socket) | +| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | +| No way out of the user | one login user (`agent`), in no group but its own, no sudo rule, login shell `/usr/sbin/nologin`; the distro's default user is `root`, not `agent`; cron, at, ssh and the agent's systemd user manager (`user@.service`) masked, so no login, linger or user unit of the agent runs anything outside T3's unit | +| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | +| No LAN, no Windows host | twice, nftables for the agent's uid and the unit's `IPAddressDeny=`: private, link-local, CGNAT and multicast ranges blocked but for `allow` and the linking `peers`, so the private addresses of the Windows host, the home network and the tailnet are out of reach; public addresses go through (the home's own public address too: see Limits) | +| Localhost only for T3 | the unit's `SocketBindDeny=any` with `SocketBindAllow=` T3's port and `devPorts`, and nftables dropping the agent's other localhost traffic: the agent listens on, and reaches on `localhost`, only T3's port and `devPorts`, so servers of other distros on `localhost` are out of reach | +| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked; T3 itself is root's (under `/opt/pc-agents/t3`) | +| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and every 2 minutes (a failed wall stops T3; a check that cannot run or crashes stops it too) | + +`pc-agents check` is the fleet's WSL2 walls check (`.github/scripts/fork-fleet-walls.py` with +`fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: + +- every file above is root's, mode 644 and exactly what the kit writes (and recorded in the root-owned + `/etc/pc-agents/installed.tsv`); +- nothing overrides the kit's units: no drop-in or other copy in any folder systemd reads (`system.control`, + `system.attached`, the generators, `/run`, `/usr/lib`, prefix drop-ins like `t3code-.service.d`), and + `systemctl show` agrees (the kit's file, no drop-in, no pending reload, T3 and the timer enabled); +- the masks are in place, no user lingers, the agent has no systemd user units, the timer is running; +- `agent` is the only login user, its login shell is `/usr/sbin/nologin`, and no sudo rule (`/etc/sudoers`, + `/etc/sudoers.d`) names it, its group or uid, or every user; +- the kit's code (`/opt/pc-agents`, the release folder it runs) and T3's runtime are root's, and T3's `t3` is the one + setup installed (its hash is in the record); +- no Windows drive is mounted; +- the loaded firewall table is the one from the kit's `firewall.nft`: `pc-agents load-firewall` saves the file's hash + and what the kernel then lists, and any later change, or an older file left loaded, differs; +- the default gateway (the Windows host) is in a blocked range, and no `allow` or `peers` entry lets it through. + +The fleet's `mounts`, `sockets` and `userns` walls are +about the CI launcher's job units, so they are not reported here; the unit's own sandbox is checked instead (it must +be the kit's file, byte for byte, with no drop-in). + +Linux (WSL2, systemd, Debian or Ubuntu) only. Python 3 standard library only. This folder has the walls and their +check so far; `pc-agents setup`, which writes them into a fresh distro, and its owner steps are still to come. + +## Config + +The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine.json` (root-owned, see +[common](../common/README.md)). Example (made-up values): + +```json +{ + "version": 1, + "pc-agents": { + "organization": "Example home lab", + "t3Version": "0.0.30", + "t3ReleaseBaseUrl": "https://github.com/example/t3code/releases/download", + "port": 3773, + "maxRuntimeMode": "full-access", + "lockedSettings": { "agentSlots.enabled": true, "agentSlots.maxPerMachine": 2 }, + "memoryMax": "8G", + "cpuQuota": "400%", + "nameservers": ["1.1.1.1", "9.9.9.9"], + "peers": ["100.101.102.103/32"], + "tailscaleTag": "tag:pc-agents" + } +} +``` + +| Key | What | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `organization` | the name the managed policy shows in T3's settings | +| `t3Version` | the exact T3 version to run (the same as the T3 that links to this PC); installed with `scripts/install.sh` | +| `t3ReleaseBaseUrl` | where that version's release files are (`T3CODE_RELEASE_BASE_URL` of `scripts/install.sh`) | +| `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | +| `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | +| `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | +| `lockedSettings` | more T3 settings to lock, `{path: value}`: the child ceiling (`agentSlots.…`), see below | +| `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | +| `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | +| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default; never the Windows host | +| `devPorts` | optional: the TCP ports the agent's dev servers may listen on and be reached on, `LOW-HIGH` (default `4100-4199`) | +| `peers` | the linking T3's Tailscale address(es) (CIDR list, 1 to 16): the only private addresses the unit talks to | +| `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | + +Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`, +`usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's +(`apps/server/src/policy/managedSettings.ts`), and a test proves it for the fork's keys. Lock one field per dotted +path (`agentSlots.maxPerMachine`), or give a whole setting (`agentSlots`) or one record entry +(`providerInstances.codex-work`) as an object holding every field: T3 takes an object only as a complete value. The kit +refuses what is cheap to tell (a field of a provider instance, an object deeper than a record entry, the kit's own +two settings, a blank `organization`); T3 refuses the rest, like a field inside a setting that can be empty (lock the +whole setting), says why in its log (`journalctl -u t3code-agents`) and does not start. T3 exits 1 on a policy it +cannot apply, the same code as any other failure, so the unit cannot tell it apart (`RestartPreventExitStatus=`); +it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). + +A wall that fails stops T3, and nothing starts it again by itself: fix what `pc-agents check` names (or run +`pc-agents setup` again), then start it as root: `systemctl start t3code-agents.service` (its own start runs the +check first and refuses on a failed wall). + +## Limits + +- **One VM for all distros.** WSL2 runs every distro in one VM: one kernel, one network namespace, one memory cap. + The firewall table is therefore global and matches the agent's uid in every distro (keep `agentUid` apart from + the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable on + T3's port or `devPorts` (the agent's other localhost traffic is dropped). **Run no localhost servers in other + distros** of this PC on those ports. What the agent serves on those ports is forwarded to Windows' `localhost` by + WSL: open it in a browser only as you would an unknown site (a page on `localhost` shares cookies with every other + `localhost` page, whatever the port). + Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. Another + distro's root can drop or change the global nft table: the check sees it within 2 minutes and stops T3, and the + unit's own `IPAddressDeny=` (per cgroup, not global) holds meanwhile. Abstract unix sockets are shared by every + distro too (the unit cannot hide them; `RestrictAddressFamilies` keeps `AF_UNIX` for T3 itself), so a program in + another distro listening on one is reachable. +- **The policy binds an honest T3:** it is T3 that applies the locks. The kit makes sure the T3 that runs is the one + root installed (root-owned under `/opt/pc-agents/t3`, its hash recorded and checked), not one the agent could + change. An agent can still start its own programs (even its own copy of T3) inside the unit: those ignore the + policy, but stay inside every other wall. +- **Public internet is open** to the agent: the firewall blocks private ranges, not the public internet (GitHub's + and Anthropic's addresses are many and change). Add an outbound proxy if that matters. That includes the home's + own public address: a router that loops it back (hairpin NAT) lets the agent reach its admin page, if the router + serves it there, and every port-forwarded device of the home. Forward no ports to devices the agent must not + reach, and keep the router's admin page off the public side. +- **Servers in the agent's dev range only:** a test or dev server that listens on a port outside `devPorts` cannot + be reached over `localhost` (port 0, any free port, may be bound, but the agent cannot connect to it there). + Point them at `devPorts`, or widen it. +- **Shared with root's programs in this distro:** the system D-Bus (`/run/dbus`, its own access rules decide), + `/dev/shm` (mode 1777; others' files keep their modes), and `NETLINK_SOCK_DIAG`, which lists every distro's + sockets (addresses and ports, not their data). +- **DNS** goes to the public `nameservers` only. A `systemd-resolved` stub (`127.0.0.53`), which may resolve LAN + names through Windows, is out of reach: the agent's localhost traffic outside T3's port and `devPorts` is dropped. +- **Terminals:** what the agent prints in a terminal can use escape codes, such as OSC 52, which some terminals turn + into a write to the Windows clipboard. Do not paste from the clipboard blindly after watching agent output. +- **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not + run; the distro is the sandbox, so use full access. +- **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, + but the code files are not hashed (T3's `t3` is). + +## Tests + +``` +python3 -I -B fork-kit/pc-agents/tests/check.test.py +``` + +A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake `nft` and +`systemctl` (it keeps which units are enabled and running): every wall passes on a walled distro, and each one +fails on its own when its part is taken away. +`apps/server/src/policy/managedSettings.forkKeys.test.ts` proves the managed policy locks the fork's settings. diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py new file mode 100644 index 0000000000000..930ddf4e4c1e6 --- /dev/null +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -0,0 +1,380 @@ +"""pc-agents: the facts, the files this kit owns and the checks, shared by `pc-agents setup` and `pc-agents check`. + +Every path is read and written under ROOT, which is '' (the real machine) except in the tests, where +PC_AGENTS_ROOT names a fake one; root never gets a fake one. Standard library only.""" +import hashlib, ipaddress, json, os, re, signal, subprocess + +import machine_file + +KIT = 'pc-agents' +ROOT = os.environ.get('PC_AGENTS_ROOT', '') +if ROOT and os.geteuid() == 0: + raise SystemExit('pc-agents: PC_AGENTS_ROOT is for the tests only, never for root; unset it') +OWNER = os.geteuid() if ROOT else 0 # who must own every file the kit writes +HOME, USER = '/home/agent', 'agent' +NOLOGIN = '/usr/sbin/nologin' # the agent's login shell: no shell for `wsl -u agent` or su; T3 gets SHELL from its unit +OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' +RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote (a path may have two mid-write) +LOADED = '/run/pc-agents/firewall.loaded' # sha256 of the firewall file and `nft list` of it, saved when loaded +T3 = OPT + '/t3' # T3's runtime, installed by root (T3CODE_HOME of scripts/install.sh): the agent cannot change it +UNIT, WALLS = 't3code-agents.service', 'pc-agents-walls' +UNITS = (UNIT, WALLS + '.service', WALLS + '.timer') +CHECK = OPT + '/current/sbin/pc-agents' +WALLS_PY = OPT + '/current/walls/fork-fleet-walls.py' +# Walls of fork-fleet-walls/wsl2 that are about the fleet launcher's job units. Here the job unit is T3's own unit, +# whose sandbox this kit writes and checks itself (the unit file must be exactly the kit's, with no drop-in). +UNIT_WALLS = {'mounts', 'sockets', 'userns'} +MASKS = ['/etc/binfmt.d/WSLInterop.conf'] + ['/etc/systemd/system/' + u for u in + ('cron.service', 'atd.service', 'ssh.service', 'ssh.socket')] +# Every folder systemd reads system units and drop-ins from (systemd.unit(5)); the kit's units live in the first. +UNIT_DIRS = ('/etc/systemd/system', '/etc/systemd/system.control', '/run/systemd/system.control', + '/run/systemd/transient', '/run/systemd/generator.early', '/etc/systemd/system.attached', + '/run/systemd/system', '/run/systemd/system.attached', '/run/systemd/generator', + '/usr/local/lib/systemd/system', '/usr/lib/systemd/system', '/lib/systemd/system', + '/run/systemd/generator.late') +BLOCKED4 = ('0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '169.254.0.0/16', '172.16.0.0/12', '192.168.0.0/16', + '224.0.0.0/4', '240.0.0.0/4') +BLOCKED6 = ('fc00::/7', 'fe80::/10', 'ff00::/8') +MODES = ('approval-required', 'auto-accept-edits', 'auto', 'full-access') +KIT_SETTINGS = {'linkedEnvironmentsEnabled': True, 'automaticUpdates': False} +VERSION = re.compile(r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?') +HEAD = '# pc-agents kit: written by `pc-agents setup`; an edit is refused at its next run and fails the walls check\n' + +# What the kit runs from a release: source path -> (path in the version folder, mode). The walls are the merged +# fleet walls of the same release, not a copy: fork-fleet-walls.py reads its folder and the launcher beside it. +CODE = {'fork-kit/pc-agents/sbin/pc-agents': ('sbin/pc-agents', 0o755), + 'fork-kit/pc-agents/lib/pc_agents.py': ('lib/pc_agents.py', 0o644), + 'fork-kit/common/machine_file.py': ('lib/machine_file.py', 0o644), + '.github/scripts/fork-fleet-walls.py': ('walls/fork-fleet-walls.py', 0o755), + '.github/scripts/fork-fleet-launch.sh': ('walls/fork-fleet-launch.sh', 0o755), + '.github/scripts/fork-fleet-job-paths.txt': ('walls/fork-fleet-job-paths.txt', 0o644), + 'scripts/install.sh': ('share/install.sh', 0o644)} +for _w, _m in (('linux', 0o755), ('wsl2', 0o755), ('linux.fixture.json', 0o644), ('wsl2.fixture.json', 0o644)): + CODE['.github/scripts/fork-fleet-walls/' + _w] = ('walls/fork-fleet-walls/' + _w, _m) + + +def is_ip_list(value, net, glob=False, most=64): + try: + ips = [(ipaddress.ip_network if net else ipaddress.ip_address)(v) for v in value] + except (TypeError, ValueError) as e: + return f'not a list of IP {"networks" if net else "addresses"} ({e})' + if not 0 < len(ips) <= most and not (net and not ips): + return f'1 to {most} entries' + return 'only public addresses (a private one is blocked)' if glob and not all(i.is_global for i in ips) else None + + +SPEC = { + 'organization': (str, machine_file.REQUIRED, + lambda v: None if v.strip() and len(v) <= 80 else '1 to 80 characters, not blank'), + 't3Version': (str, machine_file.REQUIRED, lambda v: None if VERSION.fullmatch(v) else 'an exact version, no v'), + 't3ReleaseBaseUrl': (str, machine_file.REQUIRED, + lambda v: None if re.fullmatch(r'https://[A-Za-z0-9./_-]+', v) else 'an https URL'), + 'port': (int, machine_file.REQUIRED, lambda v: None if 1024 <= v <= 65535 else '1024 to 65535'), + 'agentUid': (int, 1100, lambda v: None if 1000 <= v < 60000 else '1000 to 59999'), + 'maxRuntimeMode': (str, machine_file.REQUIRED, lambda v: None if v in MODES else 'one of ' + ', '.join(MODES)), + 'lockedSettings': (dict, {}, lambda v: next(filter(None, (locked_problem(p, x) for p, x in v.items())), None)), + 'memoryMax': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*[KMGT]', v) else 'like 8G'), + 'cpuQuota': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*%', v) else 'like 400%'), + 'nameservers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, False, True, 3)), + 'allow': (list, [], lambda v: is_ip_list(v, True)), + 'peers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, True, most=16) if v else '1 to 16 entries'), + 'tailscaleTag': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'tag:[a-z0-9-]{1,50}', v) else 'tag:name'), + 'devPorts': (str, '4100-4199', lambda v: None if port_range(v) else 'LOW-HIGH, 1024 to 65535, like 4100-4199'), +} + + +def port_range(text): + """(low, high) of a 'LOW-HIGH' port range of 1024..65535, or None.""" + m = re.fullmatch(r'([1-9]\d{3,4})-([1-9]\d{3,4})', text) + low, high = (int(m[1]), int(m[2])) if m else (0, 0) + return (low, high) if 1024 <= low <= high <= 65535 else None + + +def loopback_ports(c): + """The TCP ports the agent may listen on and reach on localhost: T3's port and devPorts (nft and the unit's + SocketBindAllow=). Written apart unless the range holds T3's port (nft refuses overlapping set elements).""" + low, high = port_range(c['devPorts']) + return ([] if low <= c['port'] <= high else [str(c['port'])]) + [f'{low}-{high}'] + + +def locked_problem(path, value): + """Why T3 would refuse to lock path to value, where that is cheap to tell (T3 checks the rest and does not start + on a policy it cannot apply: managedSettings.ts), or None. An object is a complete value: a whole setting + (`agentSlots`) or one entry of a record (`providerInstances.codex-work`), never a part of one.""" + segments = path.split('.') + if not re.fullmatch(r'[a-z]\w*(\.[A-Za-z0-9_-]+)*', path) or segments[0] in KIT_SETTINGS: + return f'{path}: set by the kit itself, or not a settings path' + if segments[0] == 'providerInstances' and len(segments) > 2: + return f'{path}: lock providerInstances.{segments[1]} as a whole (T3 refuses one field of a provider instance)' + if isinstance(value, dict) and len(segments) > 2: + return (f'{path}: an object only as a whole setting ({segments[0]}) or a record entry; lock one field per ' + 'dotted path instead (T3 does not start on a partial object)') + return None + + +def config(): + section = machine_file.read_section(KIT, ROOT + machine_file.default_path() if ROOT else None, OWNER) + return machine_file.fields(section, SPEC, f'{machine_file.default_path()}: {KIT}') + + +def runtime(c): + """T3's version folder (root's, from scripts/install.sh with T3CODE_HOME=T3); the unit runs its `t3`.""" + return f'{T3}/runtime/versions/{c["t3Version"]}' + + +def masks(c): + """Units the kit masks: WSL's binfmt handler, cron, at, ssh and the agent's systemd user manager (with it masked, + no login of the agent, no linger and no user unit runs anything outside T3's unit).""" + return MASKS + [f'/etc/systemd/system/user@{c["agentUid"]}.service'] + + +def dropin_dirs(unit): + """Every drop-in folder name systemd reads for unit: its own, its type's and each dash prefix's.""" + name, kind = unit.rsplit('.', 1) + parts = name.split('-') + return [unit + '.d', kind + '.d'] + ['-'.join(parts[:i]) + f'-.{kind}.d' for i in range(1, len(parts))] + + +def render(c): + """{path: (text, mode)} of every file the kit owns, from the config alone (the same every run).""" + settings = {**KIT_SETTINGS, **c['lockedSettings']} + policy = {'schemaVersion': 1, 'organization': c['organization'], + 'settings': {k: {'value': v, 'locked': True} for k, v in settings.items()}, + 'limits': {'maxRuntimeMode': c['maxRuntimeMode'], 'connect': False, 'webhookTasks': False, + 'browserCookieImport': False, 'updates': {'mode': 'off'}}} + allow4 = [a for a in c['allow'] if ':' not in a] + allow6 = [a for a in c['allow'] if ':' in a] + nft = HEAD + 'table inet pc_agents\ndelete table inet pc_agents\ntable inet pc_agents {\n\tchain agent_out {\n' + \ + '\t\ttype filter hook output priority filter; policy accept;\n' + \ + f'\t\tmeta skuid != {c["agentUid"]} accept\n\t\tct state established,related accept\n' + \ + f'\t\toifname "lo" tcp dport {{ {", ".join(loopback_ports(c))} }} accept\n\t\toifname "lo" drop\n' + \ + (f'\t\tip daddr {{ {", ".join(allow4)} }} accept\n' if allow4 else '') + \ + (f'\t\tip6 daddr {{ {", ".join(allow6)} }} accept\n' if allow6 else '') + \ + f'\t\tip daddr {{ {", ".join(BLOCKED4)} }} drop\n\t\tip6 daddr {{ {", ".join(BLOCKED6)} }} drop\n\t}}\n}}\n' + unit = HEAD + f'''[Unit] +Description=T3 Code for agents, walled (pc-agents kit) +Wants=network-online.target +After=network-online.target +# T3 exits 1 on a policy it cannot apply, like on any other failure: give up after 5 starts in 10 minutes. +StartLimitIntervalSec=600 +StartLimitBurst=5 + +[Service] +User={USER} +Group={USER} +Environment=HOME={HOME} T3CODE_HOME={HOME}/.t3 T3CODE_PORT={c["port"]} T3CODE_LAN=true +Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin SHELL=/bin/bash +ExecStartPre=+{CHECK} load-firewall +ExecStartPre=+{CHECK} check +ExecStart={runtime(c)}/t3 serve +Restart=on-failure +RestartSec=30 +UMask=0077 +MemoryMax={c["memoryMax"]} +CPUQuota={c["cpuQuota"]} +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths={HOME} +PrivateTmp=yes +PrivateDevices=yes +PrivateIPC=yes +ProtectProc=invisible +InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale +IPAddressDeny={" ".join(BLOCKED4 + BLOCKED6)} +IPAddressAllow={" ".join(c["peers"] + c["allow"])} +SocketBindDeny=any +{chr(10).join("SocketBindAllow=tcp:" + p for p in loopback_ports(c))} +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK +SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS +SystemCallArchitectures=native +RestrictNamespaces=yes +RestrictSUIDSGID=yes +RestrictRealtime=yes +LockPersonality=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes + +[Install] +WantedBy=multi-user.target +''' + wsl = HEAD + ('[boot]\nsystemd=true\n[automount]\nenabled=false\nmountFsTab=false\n[interop]\nenabled=false\n' + 'appendWindowsPath=false\n[network]\ngenerateResolvConf=false\n[user]\ndefault=root\n') + sd = '/etc/systemd/system/' + return { + '/etc/wsl.conf': (wsl, 0o644), + '/etc/resolv.conf': (HEAD + ''.join(f'nameserver {n}\n' for n in c['nameservers']), 0o644), + '/etc/t3code/policy.json': (json.dumps(policy, indent=2, sort_keys=True) + '\n', 0o644), + STATE + '/firewall.nft': (nft, 0o644), + sd + UNIT: (unit, 0o644), + # A check that does not succeed, even one that cannot start or crashes before it can stop T3, stops T3. + # After T3's unit: a check started while T3 is still starting (slow network at boot) waits for that start, + # instead of failing on a firewall not loaded yet and stopping T3 before it ran. + sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\nAfter={UNIT}\n\n[Service]\n' + f'Type=oneshot\nTimeoutStartSec=5min\nExecStart={CHECK} check --enforce\n' + 'ExecStopPost=/bin/sh -c \'[ "$$SERVICE_RESULT" = success ] || ' + f'exec /usr/bin/systemctl stop {UNIT}\'\n', 0o644), + sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents walls check every 2 minutes\n\n[Timer]\n' + 'OnActiveSec=1min\nOnUnitActiveSec=2min\nAccuracySec=10s\n\n[Install]\n' + 'WantedBy=timers.target\n', 0o644), + } + + +def sha(text): + return hashlib.sha256(text.encode() if isinstance(text, str) else text).hexdigest() + + +def file_sha(path): + h = hashlib.sha256() + with open(ROOT + path, 'rb') as f: + for block in iter(lambda: f.read(1 << 20), b''): + h.update(block) + return h.hexdigest() + + +def record(): + """{path: {sha256, ...}} of the record: one hash per path, two while setup replaces a file (old and new).""" + out = {} + try: + with open(ROOT + RECORD, encoding='utf-8') as f: + for line in f: + if '\t' in line: + h, path = line.rstrip('\n').split('\t', 1) + out.setdefault(path, set()).add(h) + except FileNotFoundError: + pass + return out + + +def save(path, text): + """Write a small state file whole (new file, one rename) unless it already says text.""" + real = ROOT + path + try: + with open(real, encoding='utf-8') as f: + if f.read() == text: + return + except (FileNotFoundError, UnicodeDecodeError): + pass + os.makedirs(os.path.dirname(real), mode=0o755, exist_ok=True) + with open(real + '.new', 'w', encoding='utf-8') as f: + f.write(text) + os.replace(real + '.new', real) + + +def nft_listing(): + r = run(['/usr/sbin/nft', 'list', 'table', 'inet', 'pc_agents']) + return r.stdout if r.returncode == 0 else None + + +def load_firewall(): + """Load firewall.nft and save its hash with what the kernel then lists, so the check can tell that the loaded + table is this file's (any change by anyone, or an older file left loaded, differs). Returns why not, or None.""" + body = file_sha(STATE + '/firewall.nft') + r = run(['/usr/sbin/nft', '-f', STATE + '/firewall.nft']) + listing = nft_listing() if r.returncode == 0 else None + if listing is None: + return f'cannot load {STATE}/firewall.nft: {(r.stderr or r.stdout).strip()[-300:]}' + save(LOADED, f'{body}\n{listing}') + return None + + +def lstat(path): + try: + return os.lstat(ROOT + path) + except FileNotFoundError: + return None + + +def safe(path, st, kind): + """Why path (stat st) is not what kind says (S_ISREG etc.), owned by OWNER, writable by it alone; or None.""" + if st is None: + return f'{path} is missing' + if not kind(st.st_mode) or st.st_uid != OWNER or st.st_mode & 0o022: + return f'{path} is not owned by uid {OWNER} and writable by it alone (or is the wrong kind of file)' + return None + + +def users(): + """{name: (uid, gid)} of the passwd file, and {group: [members]}.""" + def rows(path): + try: + with open(ROOT + path, encoding='utf-8') as f: + return [line.rstrip('\n').split(':') for line in f if line.count(':') >= 3] + except FileNotFoundError: + return [] + return ({r[0]: (int(r[2]), int(r[3])) for r in rows('/etc/passwd') if r[2].isdigit() and r[3].isdigit()}, + {r[0]: [m for m in r[3].split(',') if m] for r in rows('/etc/group')}) + + +def login_shell(name): + """name's login shell in the passwd file, or None.""" + try: + with open(ROOT + '/etc/passwd', encoding='utf-8') as f: + return next((r[6] for r in (line.rstrip('\n').split(':') for line in f) if r[0] == name and len(r) > 6), + None) + except FileNotFoundError: + return None + + +def sudo_problem(c): + """Why sudo could give the agent more (a sudoers line naming it, its group or uid, or every user), or None. + Harmless inside the unit (NoNewPrivileges), but a way out from anywhere else.""" + who = re.compile(rf'(^|[\s,=:!(])(%?{USER}|%?#{c["agentUid"]}|ALL)($|[\s,=:)])') + found = [] + names = ['/etc/sudoers'] + (['/etc/sudoers.d/' + n for n in sorted(os.listdir(ROOT + '/etc/sudoers.d'))] + if os.path.isdir(ROOT + '/etc/sudoers.d') else []) + for path in names: + try: + with open(ROOT + path, encoding='utf-8', errors='replace') as f: + lines = f.read().replace('\\\n', ' ').splitlines() + except (FileNotFoundError, IsADirectoryError): + continue + for line in lines: + line = re.sub(r'#(?!include)(?!\d).*', '', line).strip() + if line.startswith('Defaults') or line.startswith(('@include', '#include')): + continue + first = line.split(None, 1)[0] if line else '' + # a user list (first field) or an alias naming the agent; `root ALL=(ALL) ALL` names root, not the agent + if line and (who.search(first) or re.match(r'User_Alias\b', line) and who.search(line.split('=', 1)[-1])): + found.append(f'{path}: {line[:80]}') + return found and 'sudo rules that cover the agent: ' + '; '.join(found[:3]) + + +def user_problem(c, need_agent): + """Why this distro is someone else's (another login user, an agent that is not the kit's), or None.""" + people, groups = users() + others = sorted(n for n, (uid, _) in people.items() if 1000 <= uid < 60000 and n != USER) + if others: + return f'this distro already has login users ({", ".join(others)}): use a fresh distro for pc-agents' + if USER not in people: + return f'no user {USER}' if need_agent else None + if people[USER][0] != c['agentUid']: + return f'user {USER} has uid {people[USER][0]}, not {c["agentUid"]}: not the kit\'s' + extra = sorted(g for g, members in groups.items() if USER in members) + return f'user {USER} is in other groups ({", ".join(extra)}): it must have none' if extra else None + + +def run(argv, timeout=120): + """Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops, the + whole group is killed (a download's curl too); a timeout returns exit 124 and says so in stderr.""" + p = subprocess.Popen([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + out, err = p.communicate(timeout=timeout) + except BaseException as e: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + out, err = p.communicate(timeout=10) + except subprocess.TimeoutExpired: # something left the group but holds the pipes: stop reading + out, err = '', '' + if not isinstance(e, subprocess.TimeoutExpired): + raise + return subprocess.CompletedProcess(argv, 124, out, f'{err}\n{argv[0]} took more than {timeout} s; stopped it') + return subprocess.CompletedProcess(argv, p.returncode, out, err) diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents new file mode 100755 index 0000000000000..188d680d13a44 --- /dev/null +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -0,0 +1,175 @@ +#!/usr/bin/python3 -I +"""pc-agents: wall a WSL2 distro in, then run T3 Code in it as an unprivileged user (fork-kit/pc-agents/README.md). + + pc-agents check [--enforce] every wall, PASS or FAIL; exit 0 only when all hold. --enforce also + stops T3 when one fails (the timer, every 2 minutes). T3's unit runs + it before every start, so T3 never starts on a failed wall. + pc-agents load-firewall as root: load the agent firewall and note what the kernel lists +Standard library only.""" +import os, re, stat, sys + +HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +LIB = os.path.join(HERE, 'lib') +COMMON = os.path.join(os.path.dirname(HERE), 'common') # the repository's; an installed kit has its own copy in LIB +sys.path[:0] = [LIB] + ([] if os.path.exists(os.path.join(LIB, 'machine_file.py')) else [COMMON]) +sys.dont_write_bytecode = True +import pc_agents as k # noqa: E402 + + +def say(text): + print(f'pc-agents: {text}', flush=True) + + +def tree(top, inside=None): + """Why something in the folder top (itself included) is not root's alone, or a link out of inside; or None.""" + bad = [why] if (why := k.safe(top, k.lstat(top), stat.S_ISDIR)) else [] + for here, dirs, names in os.walk(k.ROOT + top): + for n in dirs + names: + p = os.path.join(here, n)[len(k.ROOT):] + st = k.lstat(p) + if stat.S_ISLNK(st.st_mode) and inside and st.st_uid == k.OWNER and \ + os.path.realpath(k.ROOT + p).startswith(os.path.realpath(k.ROOT + inside) + '/'): + continue + bad += [why] if (why := k.safe(p, st, lambda m: stat.S_ISDIR(m) or stat.S_ISREG(m))) else [] + return bad[:3] and '; '.join(bad[:3]) + + +def own_walls(c, before_enable=False): + """(wall, what, why or None) for the walls this kit adds to fork-fleet-walls/wsl2. before_enable: setup asks + before it enables T3 and the timer, so they need not be enabled yet.""" + out, done = [], k.record() + for path, (text, mode) in k.render(c).items(): + st = k.lstat(path) + why = k.safe(path, st, stat.S_ISREG) + if why is None: + with open(k.ROOT + path, 'rb') as f: + why = None if f.read() == text.encode() and k.sha(text) in done.get(path, ()) else \ + f'{path} differs from what the kit writes (pc-agents setup)' + out.append(('file:' + os.path.basename(path), f'{path} is the kit\'s, root-owned', why)) + out.append(('record', f'{k.RECORD} is root-owned', k.safe(k.RECORD, k.lstat(k.RECORD), stat.S_ISREG))) + masks = [p for p in k.masks(c) if not (k.lstat(p) and os.path.islink(k.ROOT + p) + and os.readlink(k.ROOT + p) == '/dev/null')] + out.append(('masks', 'WSL interop, cron, at, ssh and the agent\'s user manager are masked', + masks and 'not masked: ' + ', '.join(masks))) + linger = sorted(os.listdir(k.ROOT + '/var/lib/systemd/linger')) if k.lstat('/var/lib/systemd/linger') else [] + out.append(('linger', 'no user lingers', linger and 'lingering: ' + ', '.join(linger[:5]))) + units = [p for p in ('.config/systemd/user', '.config/systemd/user.control', '.local/share/systemd/user') + if os.path.isdir(f'{k.ROOT}{k.HOME}/{p}') and os.listdir(f'{k.ROOT}{k.HOME}/{p}')] + out.append(('user-units', f'{k.USER} has no systemd user units', units and 'found ~/' + ', ~/'.join(units))) + extra = [f'{d}/{n}' for d in k.UNIT_DIRS for u in k.UNITS + for n in k.dropin_dirs(u) + ([u] if d != k.UNIT_DIRS[0] else []) if k.lstat(f'{d}/{n}')] + out.append(('drop-ins', 'no drop-in or other copy of the kit\'s units', extra and 'found ' + ', '.join(extra[:5]))) + seen = [] + for u in k.UNITS: # what systemd has loaded, not only what is on disk + r = k.run(['/usr/bin/systemctl', 'show', '-p', + 'LoadState,FragmentPath,DropInPaths,UnitFileState,NeedDaemonReload,ActiveState', u]) + got = dict(line.split('=', 1) for line in r.stdout.splitlines() if '=' in line) + want = {'LoadState': 'loaded', 'FragmentPath': f'{k.UNIT_DIRS[0]}/{u}', 'DropInPaths': '', + 'NeedDaemonReload': 'no'} + if u != k.WALLS + '.service' and not before_enable: + want['UnitFileState'] = 'enabled' + if u == k.WALLS + '.timer' and not before_enable: # enabled is not enough: a stopped timer checks nothing + want['ActiveState'] = 'active' + seen += [f'{u}: {key}={got.get(key)!r}' for key, v in want.items() if r.returncode or got.get(key) != v] + out.append(('units', 'systemd runs the kit\'s units as written, enabled, the timer running', + seen and '; '.join(seen[:3]))) + shell = k.login_shell(k.USER) + out.append(('agent', f'one login user, {k.USER}, in no group but its own, with no login shell', + k.user_problem(c, True) or (None if shell == k.NOLOGIN else + f'{k.USER}\'s login shell is {shell}, not {k.NOLOGIN}'))) + out.append(('sudoers', f'no sudo rule covers {k.USER}', k.sudo_problem(c))) + current = k.lstat(k.OPT + '/current') + target = os.readlink(k.ROOT + k.OPT + '/current') if current and stat.S_ISLNK(current.st_mode) else '' + code = [] if current and current.st_uid == k.OWNER and re.fullmatch(r'[0-9a-f]{40}', target) else \ + [f'{k.OPT}/current is not the kit\'s link to a release folder'] + code += [why for d in ('/', '/opt', k.OPT) if (why := k.safe(d, k.lstat(d), stat.S_ISDIR))] + code += [why] if target and (why := tree(f'{k.OPT}/{target}')) else [] + out.append(('kit-code', f'{k.OPT} and the release it runs are root\'s alone', code[:3] and '; '.join(code[:3]))) + folder, rt = k.runtime(c), [] + if k.lstat(folder): # else not installed (yet): T3 cannot start + rt = [why for d in (k.T3, k.T3 + '/runtime', k.T3 + '/runtime/versions') + if (why := k.safe(d, k.lstat(d), stat.S_ISDIR))] or [why for why in [tree(folder, folder)] if why] + if not rt and k.lstat(folder + '/t3') and k.file_sha(folder + '/t3') not in done.get(folder + '/t3', ()): + rt = [f'{folder}/t3 is not the one pc-agents setup installed'] + out.append(('runtime', f'T3 {c["t3Version"]} is root\'s alone, as installed', rt and '; '.join(rt))) + mounts = [] + try: + with open(k.ROOT + '/proc/self/mountinfo', encoding='utf-8', errors='replace') as f: + for line in f: + fields = line.split(' - ') + where, fstype = fields[0].split()[4], (fields[1].split() or [''])[0] if len(fields) > 1 else '' + if fstype in ('9p', 'drvfs', 'virtiofs') and not (where + '/').startswith('/usr/lib/wsl/'): + mounts.append(f'{where} ({fstype})') + except (OSError, IndexError) as e: + mounts.append(f'cannot read /proc/self/mountinfo ({e})') + out.append(('automount', 'no Windows drive is mounted', mounts and 'mounted: ' + ', '.join(mounts[:3]))) + saved = None + if k.safe(k.LOADED, k.lstat(k.LOADED), stat.S_ISREG) is None: + with open(k.ROOT + k.LOADED, encoding='utf-8', errors='replace') as f: + saved = f.read() + listing = k.nft_listing() + fw = 'table inet pc_agents is not loaded' if listing is None else None if saved == \ + f'{k.sha(k.render(c)[k.STATE + "/firewall.nft"][0])}\n{listing}' else \ + 'the loaded table inet pc_agents is not the one from the kit\'s firewall.nft (pc-agents load-firewall)' + out.append(('firewall', 'the kit\'s agent firewall is what is loaded', fw)) + gateway = None + try: + with open(k.ROOT + '/proc/net/route', encoding='utf-8') as f: + row = next((line.split() for line in f if line.split()[1:2] == ['00000000']), None) + gateway = row and k.ipaddress.ip_address(bytes.fromhex(row[2])[::-1]) + except (OSError, ValueError) as e: + gateway = f'unknown ({e})' + + def nets(names): + return [n for n in names if isinstance(gateway, k.ipaddress.IPv4Address) and gateway in k.ipaddress.ip_network(n)] + opened = nets(c['allow'] + c['peers']) + out.append(('gateway', 'the Windows host (the default gateway) is in a blocked range and not let through', + f'allow or peers let the gateway {gateway} through: {", ".join(opened)}' if opened else + None if nets(k.BLOCKED4) else f'the gateway is {gateway}, not in a blocked range')) + return out + + +def wsl2_walls(): + """(wall, what, why or None) of the fleet's wsl2 walls (fork-fleet-walls.py, its contract enforced there).""" + env = {'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'LC_ALL': 'C'} + if k.ROOT: + env['FORK_FLEET_WALLS_ROOT'] = k.ROOT + try: + p = k.subprocess.run([k.ROOT + k.WALLS_PY, 'describe'], env=env, stdin=k.subprocess.DEVNULL, + capture_output=True, text=True, timeout=120, cwd='/') + except (OSError, k.subprocess.TimeoutExpired) as e: + return [('wsl2', 'the fleet\'s wsl2 walls ran', f'cannot run them ({e})')] + lines = [m.groups() for m in re.finditer(r'^(PASS|FAIL) (\S+) (.*)$', p.stdout, re.M)] + if p.returncode not in (0, 1) or not lines: + return [('wsl2', 'the fleet\'s wsl2 walls ran', f'exit {p.returncode}: {p.stderr.strip()[-300:]}')] + return [(f'wsl2:{w}', what, None if mark == 'PASS' else what) for mark, w, what in lines if w not in k.UNIT_WALLS] + + +def check(enforce, before_enable=False): + try: + c = k.config() + results = wsl2_walls() + own_walls(c, before_enable) + except Exception as e: # a wall that cannot be told does not hold + results = [('check', 'the walls can be read', f'{e.__class__.__name__}: {e}')] + for wall, what, why in results: + print(f'FAIL {wall} {what}: {why}'[:500] if why else f'PASS {wall} {what}', flush=True) + failed = any(why for _, _, why in results) + if failed and enforce: + r = k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) + say(f'a wall does not hold: stopped {k.UNIT}' if r.returncode == 0 else + f'a wall does not hold, and stopping {k.UNIT} FAILED: {r.stderr.strip()[-300:]}') + return 1 if failed else 0 + + +def main(argv): + if argv[:1] == ['check'] and argv[1:] in ([], ['--enforce']): + return check(argv[1:] == ['--enforce']) + if argv == ['load-firewall']: + why = k.load_firewall() + return say(why) or 1 if why else 0 + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py new file mode 100644 index 0000000000000..de9717f11eec8 --- /dev/null +++ b/fork-kit/pc-agents/tests/check.test.py @@ -0,0 +1,329 @@ +"""pc-agents check on a fake WSL2 distro (fake_distro.py): every wall passes on a walled distro, and each one fails +on its own when its part of the wall is taken away; --enforce stops T3. + + python3 -I -B fork-kit/pc-agents/tests/check.test.py +""" +import os, shutil, subprocess, sys, time, unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from fake_distro import CONFIG, Case, DRVFS, SHA1, k # noqa: E402 + + +class Walls(Case): + def setUp(self): + super().setUp() + self.walled() + + def fails(self, wall, *also): + """check fails on wall, and on no other wall but those in also (which the same change takes away).""" + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertEqual({line.split()[1] for line in out.splitlines() if line.startswith('FAIL')}, {wall, *also}, out) + + def test_all_hold(self): + code, out = self.check() + self.assertEqual(code, 0, out) + walls = [line.split()[1] for line in out.splitlines()] + for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', + 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'sudoers', 'kit-code', 'runtime', + 'automount', 'firewall', 'gateway', 'file:policy.json', 'file:t3code-agents.service', + 'file:firewall.nft', 'file:wsl.conf', 'file:pc-agents-walls.timer'): + self.assertIn(wall, walls) + for wall in k.UNIT_WALLS: # the launcher's job units: not T3's unit + self.assertNotIn('wsl2:' + wall, walls) + + def test_enforce_stops_t3(self): + os.unlink(self.path('/log/nft-loaded')) + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('stop t3code-agents.service', self.log()) + self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) + self.assertIn('stopped t3code-agents.service', out) + + def test_enforce_says_when_the_stop_fails(self): + os.unlink(self.path('/log/nft-loaded')) + self.write('/log/systemctl-fail', '') + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('stopping t3code-agents.service FAILED', out) + + def test_each_wall(self): + def edit(p, old, new): + return lambda: self.write(p, self.read(p).replace(old, new)) + cases = { + 'file:policy.json': edit('/etc/t3code/policy.json', '"full-access"', '"auto"'), + 'file:t3code-agents.service': lambda: os.chmod(self.path('/etc/systemd/system/t3code-agents.service'), + 0o666), + 'file:resolv.conf': lambda: os.unlink(self.path('/etc/resolv.conf')), + 'masks': lambda: os.unlink(self.path('/etc/systemd/system/ssh.socket')), + 'drop-ins': lambda: os.makedirs(self.path('/etc/systemd/system/t3code-agents.service.d')), + 'agent': edit('/etc/group', 'sudo:x:27:', 'sudo:x:27:agent'), + 'kit-code': lambda: os.chmod(self.path(f'/opt/pc-agents/{SHA1}/lib/pc_agents.py'), 0o666), + 'automount': lambda: self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo') + DRVFS), + 'firewall': lambda: os.unlink(self.path('/log/nft-loaded')), + 'gateway': lambda: self.write('/proc/net/route', 'Iface\tDestination\tGateway\nx\t00000000\t08080808\n'), + 'wsl2:interop': lambda: self.write('/proc/sys/fs/binfmt_misc/WSLInterop', + 'enabled\ninterpreter /init\nflags: PF\noffset 0\nmagic 4d5a\n'), + 'wsl2:networking': edit('/usr/bin/wslinfo', 'echo nat', 'echo mirrored'), + 'record': lambda: os.chmod(self.path(k.RECORD), 0o666), + 'linger': lambda: self.write('/var/lib/systemd/linger/agent', ''), + 'user-units': lambda: self.write('/home/agent/.config/systemd/user/x.service', '[Service]\n'), + 'units': lambda: self.write('/log/units/t3code-agents.service.dropins', '/run/systemd/system/x.conf'), + 'runtime': lambda: os.chmod(self.path(k.runtime(CONFIG) + '/t3'), 0o777), + 'file:pc-agents-walls.timer': edit('/etc/systemd/system/pc-agents-walls.timer', '2min', '1d'), + 'sudoers': lambda: self.write('/etc/sudoers.d/90-agent', 'agent ALL=(ALL) NOPASSWD: ALL\n'), + } + clean = {p: v[4] for p, v in self.snapshot().items()} + for wall, change in cases.items(): + with self.subTest(wall=wall): + shutil.rmtree(self.root) + self.setUp() + self.assertEqual({p: v[4] for p, v in self.snapshot().items()}, clean) + change() + self.fails(wall, *(['wsl2:resolv'] if wall == 'file:resolv.conf' else [])) + + def again(self, **section): + """machine.json changed and set up again: the new files, recorded, their firewall loaded.""" + self.machine({**CONFIG, **section}) + self.files() + self.assertIsNone(k.load_firewall()) + + def test_agent_has_no_login_shell(self): + self.write('/etc/passwd', self.read('/etc/passwd').replace('/home/agent:/usr/sbin/nologin', '/home/agent:/bin/bash')) + self.fails('agent') + + def test_sudo_rules_that_cover_the_agent(self): + base = 'Defaults env_reset\nroot ALL=(ALL:ALL) ALL\n%sudo ALL=(ALL:ALL) ALL\n@includedir /etc/sudoers.d\n' + self.write('/etc/sudoers', base + '# agent ALL=(ALL) ALL is only a comment\n') + self.write('/etc/sudoers.d/README', '# agent\n') + self.assertEqual(self.check()[0], 0) + for rule in ('agent ALL=(ALL) NOPASSWD: ALL', '%agent ALL=(ALL) ALL', '#1100 ALL=(ALL) ALL', 'ALL ALL=(ALL) ALL', + 'bob,agent ALL=(ALL) ALL', 'User_Alias AGENTS = bob, agent', 'agent \\\n ALL=(ALL) ALL'): + with self.subTest(rule=rule): + self.write('/etc/sudoers.d/x', rule + '\n') + self.fails('sudoers') + os.unlink(self.path('/etc/sudoers.d/x')) + self.write('/etc/sudoers', base + 'agent ALL=(ALL) ALL\n') + self.fails('sudoers') + + def test_timer_stopped_but_enabled(self): + os.unlink(self.path('/log/units/pc-agents-walls.timer.active')) + self.fails('units') + + def test_agent_user_manager_masked(self): + os.unlink(self.path('/etc/systemd/system/user@1100.service')) + self.fails('masks') + + def test_unit_copies_and_prefix_drop_ins(self): + for p in ('/usr/lib/systemd/system/t3code-agents.service', '/run/systemd/generator.late/pc-agents-walls.timer', + '/etc/systemd/system/t3code-.service.d/', '/etc/systemd/system/pc-agents-.timer.d/', + '/run/systemd/system.attached/t3code-agents.service.d/', '/etc/systemd/system/service.d/'): + with self.subTest(path=p): + self.write(p, '[Service]\n') + self.fails('drop-ins') + (os.rmdir if p.endswith('/') else os.unlink)(self.path(p.rstrip('/'))) + + def test_units_as_systemd_loaded_them(self): + for name, change in (('timer disabled', lambda: os.unlink(self.path('/log/units/pc-agents-walls.timer.enabled'))), + ('T3 disabled', lambda: os.unlink(self.path('/log/units/t3code-agents.service.enabled'))), + ('not reloaded', lambda: self.write('/log/units/reload', '')), + ('drop-in', lambda: self.write('/log/units/pc-agents-walls.service.dropins', '/x.conf'))): + with self.subTest(name): + shutil.rmtree(self.root) + self.setUp() + change() + self.fails('units') + + def test_firewall_changed_after_it_was_loaded(self): + self.write('/log/nft-loaded', self.read('/log/nft-loaded').replace('accept', 'accept\n', 1)) + self.fails('firewall') + + def test_firewall_file_newer_than_the_loaded_table(self): + self.machine({**CONFIG, 'allow': ['192.168.1.0/24']}) + self.files() + self.fails('firewall') + self.assertIsNone(k.load_firewall()) + self.assertEqual(self.check()[0], 0) + + def test_allow_or_peers_covering_the_gateway(self): + for key in ('allow', 'peers'): + with self.subTest(key): + self.again(**{key: ['172.16.0.0/12']}) + self.fails('gateway') + self.again() + self.assertEqual(self.check()[0], 0) + + def test_runtime_changed_or_missing(self): + t3 = k.runtime(CONFIG) + '/t3' + self.write(t3, {'text': '#!/bin/sh\nexec /home/agent/x\n', 'mode': '755'}) + self.fails('runtime') + os.chmod(self.path('/opt/pc-agents/t3/runtime'), 0o777) + self.fails('runtime') + shutil.rmtree(self.path(k.runtime(CONFIG))) # not installed: T3 cannot start, nothing to wall + os.chmod(self.path('/opt/pc-agents/t3/runtime'), 0o755) + code, out = self.check() + self.assertEqual(code, 0, out) + + def test_kit_code_folders(self): + os.chmod(self.path('/opt/pc-agents'), 0o775) + self.fails('kit-code') + os.chmod(self.path('/opt/pc-agents'), 0o755) + os.symlink('/etc', self.path(f'/opt/pc-agents/{SHA1}/lib/etc')) + self.fails('kit-code') + + def test_automount_only_under_usr_lib_wsl(self): + self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo') + + '45 29 0:45 / /usr/lib/wslfoo rw - 9p drvfs rw\n') + self.fails('automount') + + def test_no_config_fails(self): + os.unlink(self.path('/etc/fork-kit/machine.json')) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL check ', out) + + +class Holes(Case): + """Ways the agent could run code outside T3's walled unit, or the walls be changed under systemd.""" + def setUp(self): + super().setUp() + self.walled() + + def test_agent_user_unit(self): + self.write('/home/agent/.config/systemd/user/x.service', '[Service]\nExecStart=/home/agent/x\n') + os.makedirs(self.path('/home/agent/.config/systemd/user/default.target.wants')) + os.symlink('../x.service', self.path('/home/agent/.config/systemd/user/default.target.wants/x.service')) + self.write('/var/lib/systemd/linger/agent', '') + self.assertEqual(self.check()[0], 1) + + def test_system_control_drop_in(self): + self.write('/etc/systemd/system.control/t3code-agents.service.d/50-x.conf', '[Service]\nRestrictAddressFamilies=\n') + self.assertEqual(self.check()[0], 1) + + def test_timer_drop_in(self): + self.write('/etc/systemd/system/pc-agents-walls.timer.d/off.conf', '[Timer]\nOnCalendar=\n') + self.assertEqual(self.check()[0], 1) + + def test_prefix_drop_in(self): + self.write('/etc/systemd/system/t3code-.service.d/x.conf', '[Service]\nUser=root\n') + self.assertEqual(self.check()[0], 1) + + +class Crash(Case): + """A check that cannot run, or crashes, still stops T3 (the walls service's ExecStopPost).""" + def setUp(self): + super().setUp() + self.walled() + + def stop_post(self, result): + line = next(x for x in k.render(k.config())['/etc/systemd/system/pc-agents-walls.service'][0].splitlines() + if x.startswith('ExecStopPost=')) + cmd = line.split('=', 1)[1].replace('$$', '$').replace('/usr/bin/systemctl', self.path('/usr/bin/systemctl')) + subprocess.run(cmd, shell=True, check=True, env={'SERVICE_RESULT': result, 'PATH': '/usr/bin:/bin'}) + + def test_missing_lib_stops_t3(self): + os.unlink(self.path(f'/opt/pc-agents/{SHA1}/lib/pc_agents.py')) + p = subprocess.run([sys.executable, '-I', self.path(f'/opt/pc-agents/{SHA1}/sbin/pc-agents'), 'check', + '--enforce'], env={'PC_AGENTS_ROOT': self.root, 'PATH': '/usr/bin:/bin'}, + capture_output=True, text=True, timeout=60) + self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertNotIn('stop t3code-agents.service', self.log()) # the check itself could not stop T3 + self.stop_post('exit-code') + self.assertIn('stop t3code-agents.service', self.log()) + self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) + + def test_success_does_not_stop_t3(self): + self.stop_post('success') + self.assertEqual(self.log(), []) + + +class Unit(Case): + """What the kit's files say: things no wall can see from outside the unit.""" + def test_unit(self): + files = k.render(k.config()) + unit = files['/etc/systemd/system/t3code-agents.service'][0] + self.assertIn('\nExecStart=/opt/pc-agents/t3/runtime/versions/1.2.3/t3 serve\n', unit) + self.assertIn(' -/run/tailscale', unit) + deny = next(x for x in unit.splitlines() if x.startswith('IPAddressDeny=')).split('=', 1)[1].split() + self.assertEqual(set(deny), set(k.BLOCKED4 + k.BLOCKED6)) + self.assertIn('\nIPAddressAllow=100.101.102.103/32\n', unit) + self.assertIn('\nStartLimitBurst=5\n', unit) + self.assertIn('\n[user]\ndefault=root\n', files['/etc/wsl.conf'][0]) + self.assertIn('\nOnUnitActiveSec=2min\n', files['/etc/systemd/system/pc-agents-walls.timer'][0]) + self.assertIn(' SHELL=/bin/bash\n', unit) # the agent's login shell is nologin; T3's shells are bash + walls = files['/etc/systemd/system/pc-agents-walls.service'][0] + for line in ('After=t3code-agents.service', 'TimeoutStartSec=5min', 'Type=oneshot'): + self.assertIn('\n' + line + '\n', walls) + + def test_loopback(self): + """The agent listens on and reaches over localhost only T3's port and devPorts: other distros' localhost + servers are out of reach, and what the agent serves is not forwarded to Windows' localhost on any port.""" + files = k.render(k.config()) + unit = files['/etc/systemd/system/t3code-agents.service'][0] + nft = files['/etc/pc-agents/firewall.nft'][0] + self.assertIn('\nSocketBindDeny=any\nSocketBindAllow=tcp:3800\nSocketBindAllow=tcp:4100-4199\n', unit) + self.assertIn('\t\toifname "lo" tcp dport { 3800, 4100-4199 } accept\n\t\toifname "lo" drop\n', nft) + self.assertNotIn('oifname "lo" accept', nft) + self.assertLess(nft.index('oifname "lo" drop'), nft.index('ip daddr {')) + self.machine({**CONFIG, 'devPorts': '3700-3899'}) # T3's port inside the range: written once (nft) + files = k.render(k.config()) + self.assertIn('\nSocketBindDeny=any\nSocketBindAllow=tcp:3700-3899\nRestrict', files['/etc/systemd/system/t3code-agents.service'][0]) + self.assertIn('tcp dport { 3700-3899 } accept', files['/etc/pc-agents/firewall.nft'][0]) + for bad in ('80-90', '5000-4000', '4100', '4100-70000', ' 4100-4199', 4100): + with self.subTest(devPorts=bad): + self.machine({**CONFIG, 'devPorts': bad}) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('devPorts', out) + + def test_loopback_rules_are_checked(self): + self.walled() + for name, change in (('unit', lambda: self.write('/etc/systemd/system/t3code-agents.service', self.read( + '/etc/systemd/system/t3code-agents.service').replace('SocketBindDeny=any\n', ''))), + ('loaded table', lambda: self.write('/log/nft-loaded', self.read('/log/nft-loaded').replace( + '\t\toifname "lo" drop\n', '')))): + with self.subTest(name): + shutil.rmtree(self.root) + self.setUp() + self.walled() + change() + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL ' + ('file:t3code-agents.service' if name == 'unit' else 'firewall'), out) + + def test_config(self): + # The kit passes these; T3 itself tells a complete object from a partial one and does not start on a partial. + ok = {'agentSlots': {'maxPerMachine': 2}, 'providerInstances.codex-work': {'driver': 'codex'}, + 'agentSlots.maxPerMachine': 2, 'providerInstances': {}} + self.assertEqual([p for p, v in ok.items() if k.locked_problem(p, v)], []) + bad = {'providerInstances.codex-work.enabled': True, 'agentSlots.a.b': {'x': 1}, 'automaticUpdates': True, + 'linkedEnvironmentsEnabled.x': 1, 'Agent': 1, 'a..b': 1} + self.assertEqual([p for p, v in bad.items() if not k.locked_problem(p, v)], []) + for section in ({**CONFIG, 'organization': ' '}, {**CONFIG, 'peers': []}, + {key: v for key, v in CONFIG.items() if key != 'peers'}): + with self.subTest(section=section): + self.machine(section) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL check ', out) + + +class Run(Case): + def test_timeout_kills_the_whole_group(self): + self.write('/x', {'text': f'#!/bin/sh\nsleep 60 &\necho $! >{self.root}/pid\nwait\n', 'mode': '755'}) + r = k.run(['/x'], timeout=1) + self.assertEqual(r.returncode, 124) + self.assertIn('took more than 1 s', r.stderr) + pid = int(self.read('/pid')) + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail(f'sleep {pid} still runs') + + +if __name__ == '__main__': + unittest.main() diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py new file mode 100644 index 0000000000000..eab2f08c38cc6 --- /dev/null +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -0,0 +1,192 @@ +"""A fake WSL2 distro in a temporary folder for the pc-agents tests: every fixed path under PC_AGENTS_ROOT, the +current user as root, fake useradd, env (T3's installer), systemctl, systemd-run and nft (each logs to /log in the +fake root; systemctl keeps which units are enabled and which ExecStart runs in /log/units). No root, no systemd, no +network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" +import contextlib, importlib.machinery, importlib.util, io, json, os, shutil, stat, sys, tempfile, unittest + +KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +REPO = os.path.dirname(os.path.dirname(KIT)) +WSL2 = os.path.join(REPO, '.github', 'scripts', 'fork-fleet-walls', 'wsl2.fixture.json') +SHA1, SHA2 = 'a' * 40, 'b' * 40 +CONFIG = {'organization': 'Example lab', 't3Version': '1.2.3', 't3ReleaseBaseUrl': 'https://example.com/releases', + 'port': 3800, 'maxRuntimeMode': 'full-access', 'lockedSettings': {'agentSlots.maxPerMachine': 2}, + 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'peers': ['100.101.102.103/32'], + 'tailscaleTag': 'tag:pc-agents'} +TOOLS = { # fake programs, under the fake root; each finds the root from its own path + 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; sh=/bin/sh; prev=; for a; do [ "$prev" = --uid ] && uid=$a; ' + '[ "$prev" = --shell ] && sh=$a; prev=$a; done\n' + 'echo "$a:x:$uid:$uid::/home/$a:$sh" >>"$R/etc/passwd"\n' + 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', + # T3's installer, run by root through env -i (scripts/install.sh with T3CODE_HOME=/opt/pc-agents/t3) + 'usr/bin/env': 'R="${0%/usr/bin/env}"; echo "$*" >>"$R/log/install"\n[ -e "$R/log/install-fail" ] && exit 1\n' + 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' + 'd="$R/opt/pc-agents/t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n# t3 $v\\n" >"$d/t3"\n' + 'chmod 755 "$d/t3"\n', + 'usr/bin/systemd-run': 'R="${0%/usr/bin/systemd-run}"; printf "%s\\n" "$@" >"$R/log/systemd-run"\n', + 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"; S="$R/log/units"; mkdir -p "$S"\n' + '[ -e "$R/log/systemctl-fail" ] && { echo "systemctl: failed (test)" >&2; exit 1; }\n' + 'run() { grep "^ExecStart=" "$R/etc/systemd/system/$1" >"$S/$1.active" 2>/dev/null || :; }\n' + 'case "$1" in\n' + 'enable|disable) v=$1; shift; now=; [ "$1" = --now ] && { now=1; shift; }\n' + ' for u; do if [ $v = enable ]; then : >"$S/$u.enabled"; [ -n "$now" ] && ' + '{ [ -e "$S/$u.active" ] || run "$u"; }\n' + ' else rm -f "$S/$u.enabled"; [ -z "$now" ] || rm -f "$S/$u.active"; fi; done;;\n' + 'restart) shift; for u; do run "$u"; done;;\n' + 'stop) shift; for u; do rm -f "$S/$u.active"; done;;\n' + 'is-active) [ -e "$S/$2.active" ] && echo active || { echo inactive; exit 3; };;\n' + 'show) u=$4; f=; [ -e "$R/etc/systemd/system/$u" ] && f=/etc/systemd/system/$u\n' + ' st=disabled; [ -e "$S/$u.enabled" ] && st=enabled\n' + ' [ "$u" = pc-agents-walls.service ] && st=static\n' + ' echo "LoadState=${f:+loaded}"; echo "FragmentPath=$f"\n' + ' echo "DropInPaths=$(cat "$S/$u.dropins" 2>/dev/null)"; echo "UnitFileState=$st"\n' + ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no\n' + ' [ -e "$S/$u.active" ] && echo ActiveState=active || echo ActiveState=inactive;;\n' + 'esac\n', + 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' + 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', +} +DRVFS = '43 29 0:43 / /mnt/c rw,noatime - 9p drvfs rw\n44 29 0:44 / /c rw,noatime - 9p drvfs rw\n' + + +def blob(path): + with open(path, 'rb') as f: + return f.read() + + +def load(): + os.environ['PC_AGENTS_ROOT'] = '/nonexistent' + loader = importlib.machinery.SourceFileLoader('pc_agents_cli', os.path.join(KIT, 'sbin', 'pc-agents')) + mod = importlib.util.module_from_spec(importlib.util.spec_from_loader('pc_agents_cli', loader)) + loader.exec_module(mod) + return mod + + +cli = load() +k = cli.k + + +class Case(unittest.TestCase): + def setUp(self): + self.umask = os.umask(0o022) + self.root = os.path.realpath(tempfile.mkdtemp(prefix='pc-agents-test-')) + os.chmod(self.root, 0o755) + self.addCleanup(self.close) + k.ROOT, k.OWNER = self.root, os.geteuid() + with open(WSL2, encoding='utf-8') as f: + files = json.load(f)['pass'] + for rel in ('etc/wsl.conf', 'etc/resolv.conf', 'etc/binfmt.d/WSLInterop.conf'): + del files[rel] # a fresh distro: the kit writes these + files['etc/wsl.conf'] = '[boot]\nsystemd=true\n' + files['etc/passwd'] = 'root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534::/nonexistent:/usr/sbin/nologin\n' + files['etc/group'] = 'root:x:0:\nsudo:x:27:\nnogroup:x:65534:\n' + files['proc/net/route'] = 'Iface\tDestination\tGateway\tFlags\nboth\t00000000\t011014AC\t0003\n' + files['log/'] = '' + files.update({rel: {'text': '#!/bin/sh\n' + text, 'mode': '755'} for rel, text in TOOLS.items()}) + for rel, value in files.items(): + self.write('/' + rel, value) + os.symlink('/mnt/wsl/resolv.conf', self.path('/etc/resolv.conf')) + self.machine(CONFIG) + + def close(self): + os.umask(self.umask) + shutil.rmtree(self.root, ignore_errors=True) + + def path(self, p): + return self.root + p + + def write(self, p, value): + full = self.path(p) + if os.path.islink(full): + os.unlink(full) + if p.endswith('/'): + return os.makedirs(full, exist_ok=True) + os.makedirs(os.path.dirname(full), exist_ok=True) + text, mode = (value['text'], int(value['mode'], 8)) if isinstance(value, dict) else (value, 0o644) + with open(full, 'w', encoding='utf-8') as f: + f.write(text) + os.chmod(full, mode) + + def read(self, p): + with open(self.path(p), encoding='utf-8') as f: + return f.read() + + def machine(self, section): + self.write('/etc/fork-kit/machine.json', json.dumps({'version': 1, 'pc-agents': section})) + + def terminate(self): + """What `wsl --terminate` does once wsl.conf turned automount off: Windows' drives are no longer mounted.""" + self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo').replace(DRVFS, '')) + + def call(self, fn, *argv): + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + code = fn(*argv) + return code, out.getvalue() + + def setup_run(self, sha=SHA1): + return self.call(cli.main, ['setup', '--source', REPO, '--sha', sha]) + + def check(self, *flags): + return self.call(cli.main, ['check', *flags]) + + def log(self, name='systemctl'): + try: + return self.read('/log/' + name).splitlines() + except FileNotFoundError: + return [] + + def ready(self): + """A distro set up all the way: first run, `wsl --terminate`, second run.""" + code, out = self.setup_run() + self.assertEqual(code, 3, out) + self.terminate() + code, out = self.setup_run() + self.assertEqual(code, 0, out) + + def files(self): + """The kit's files from machine.json, T3's runtime and the record of both, as setup writes them.""" + c = k.config() + files = k.render(c) + for p, (text, mode) in files.items(): + self.write(p, {'text': text, 'mode': f'{mode:o}'}) + t3 = k.runtime(c) + '/t3' + self.write(t3, {'text': '#!/bin/sh\n', 'mode': '755'}) + self.write(k.RECORD, ''.join(f'{k.sha(t)}\t{p}\n' for p, (t, _) in sorted(files.items())) + + f'{k.file_sha(t3)}\t{t3}\n') + return c + + def walled(self, sha=SHA1): + """What `pc-agents setup` leaves after `wsl --terminate` and a second run, written here directly.""" + c = self.files() + for p in k.masks(c): + os.makedirs(os.path.dirname(self.path(p)), exist_ok=True) + os.symlink('/dev/null', self.path(p)) + with open(self.path('/etc/passwd'), 'a') as f: + f.write('agent:x:1100:1100::/home/agent:/usr/sbin/nologin\n') + with open(self.path('/etc/group'), 'a') as f: + f.write('agent:x:1100:\n') + for src, (dst, mode) in k.CODE.items(): + with open(os.path.join(REPO, src), encoding='utf-8') as f: + self.write(f'/opt/pc-agents/{sha}/{dst}', {'text': f.read(), 'mode': f'{mode:o}'}) + os.symlink(sha, self.path('/opt/pc-agents/current')) + self.assertIsNone(k.load_firewall()) + k.run(['/usr/bin/systemctl', 'enable', '--now', k.UNIT, k.WALLS + '.timer']) + os.unlink(self.path('/log/systemctl')) + self.terminate() + + def snapshot(self): + """Every path under the fake root but the fake tools' logs: kind, mode, owner, mtime, inode, link, content.""" + out = {} + for top, dirs, names in os.walk(self.root): + for n in dirs + names: + full = os.path.join(top, n) + rel = full[len(self.root):] + if rel.startswith('/log'): + continue + st = os.lstat(full) + body = os.readlink(full) if stat.S_ISLNK(st.st_mode) else \ + blob(full) if stat.S_ISREG(st.st_mode) else None + out[rel] = (st.st_mode, st.st_uid, st.st_mtime_ns, st.st_ino, body) + return out + +