Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/fork-ci-tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ fork-kit/heavy/tests/install.test.py
fork-kit/heavy/tests/sweep.test.py
# A real systemd user manager (the step starts one for the runner user).
fork-kit/heavy/tests/heavy.test.py run by .github/workflows/fork-tests.yml
fork-kit/pc-agents/tests/check.test.py

# Upstream's tests, run where fork-tests.yml follows upstream's ci.yml.
.github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml
Expand Down
92 changes: 73 additions & 19 deletions .github/scripts/fork-ci-tests.cjs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
// Fork only: every `*.test.*` under .github/scripts, fork-ci and fork-kit must be listed in .github/fork-ci-tests.txt.
// A plain entry runs here by its extension (RUNNER); `<test> run by <file>` is run by that file,
// which must name the test and cannot be a (disabled) upstream workflow. Unlisted or runner-less
// tests fail; a listed test that is gone fails once the tested history had it, a note before.
// Fork only: every `*.test.*` under .github/scripts, fork-ci and fork-kit is listed in .github/fork-ci-tests.txt and
// runs here by extension (RUNNER), FORK_CI_TEST_JOBS at once (default: CPU count), or `run by <file>` (not an upstream
// workflow; it names the test). Unlisted or runner-less tests fail; a listed one that is gone fails once history had it.
const childProcess = require("node:child_process");
const fs = require("node:fs");
const os = require("node:os");
const path = require("node:path");

const ROOT = path.resolve(__dirname, "../..");
Expand Down Expand Up @@ -119,24 +119,74 @@ function readFile(file) {
}
}

function main() {
// Calls start(item) for every item, at most `jobs` at once, each as soon as a slot is free.
// start returns a promise; the results come back in the items' order.
async function runAll(items, jobs, start) {
const results = new Array(items.length);
let next = 0;
async function worker() {
while (next < items.length) {
const index = next++;
results[index] = await start(items[index]);
}
}
await Promise.all(Array.from({ length: Math.max(1, Math.min(jobs, items.length)) }, worker));
return results;
}

function jobCount(env = process.env) {
const jobs = Number(env.FORK_CI_TEST_JOBS);
return Number.isInteger(jobs) && jobs > 0 ? jobs : os.availableParallelism();
}

// Runs one test file, its stdout and stderr kept together, and prints them as one group when it ends.
function runTest(file) {
const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]];
const started = Date.now();
return new Promise((resolve) => {
const child = childProcess.spawn(command, [...args, file], {
cwd: ROOT,
stdio: ["ignore", "pipe", "pipe"],
});
const output = [];
child.stdout.on("data", (chunk) => output.push(chunk));
child.stderr.on("data", (chunk) => output.push(chunk));
let ended = false;
const done = (status, signal) => {
if (ended) return;
ended = true;
// Stop reading pipes a left-behind process still holds, so they do not keep this script alive.
child.stdout.destroy();
child.stderr.destroy();
const seconds = ((Date.now() - started) / 1000).toFixed(1);
process.stdout.write(`::group::${file} (${seconds} s)\n`);
process.stdout.write(Buffer.concat(output));
process.stdout.write("\n::endgroup::\n");
if (status !== 0)
console.log(`::error file=${file}::${file} failed (exit ${status ?? signal})`);
resolve({ file, status, seconds });
};
child.on("error", (error) => {
output.push(Buffer.from(`cannot start ${command}: ${error.message}\n`));
done(null, "spawn error");
});
child.on("close", done);
// A process the test left behind may hold the pipes open: the test is done when it exits.
child.on("exit", (status, signal) => setTimeout(() => done(status, signal), 5000).unref());
});
}

async function main() {
const entries = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8"));
const result = plan(entries, presentTests(), everInHistory, readFile);
const rows = [];
let failed =
result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length;
for (const file of result.run) {
const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]];
const started = Date.now();
console.log(`::group::${file}`);
const run = childProcess.spawnSync(command, [...args, file], { cwd: ROOT, stdio: "inherit" });
console.log("::endgroup::");
const seconds = ((Date.now() - started) / 1000).toFixed(1);
if (run.status !== 0) {
failed += 1;
console.log(`::error file=${file}::${file} failed (exit ${run.status ?? run.signal})`);
}
rows.push(`| \`${file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${seconds} s |`);
for (const run of await runAll(result.run, jobCount(), runTest)) {
if (run.status !== 0) failed += 1;
rows.push(
`| \`${run.file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${run.seconds} s |`,
);
}
for (const file of result.unlisted)
console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`);
Expand Down Expand Up @@ -175,6 +225,10 @@ function main() {
if (failed) process.exitCode = 1;
}

module.exports = { plan, readList, findTests, presentTests };
module.exports = { plan, readList, findTests, presentTests, runAll, jobCount };

if (require.main === module) main();
if (require.main === module)
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});
180 changes: 179 additions & 1 deletion .github/scripts/fork-ci-tests.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,14 @@ const os = require("node:os");
const path = require("node:path");
const test = require("node:test");

const { plan, readList, findTests, presentTests } = require("./fork-ci-tests.cjs");
const {
plan,
readList,
findTests,
presentTests,
runAll,
jobCount,
} = require("./fork-ci-tests.cjs");

const never = () => false;
const here = (...files) => files.map((file) => ({ file, runBy: null }));
Expand Down Expand Up @@ -143,3 +150,174 @@ test("an unlisted test under .github/scripts or fork-ci fails the run", () => {
fs.rmSync(root, { recursive: true, force: true });
}
});

test("runAll runs at most `jobs` at once, starts the next as soon as a slot frees, keeps the order", async () => {
const runs = new Map(); // item -> resolve of its run, in start order
const waiting = []; // [count, resolve]: resolved once that many runs started
const started = (count) =>
new Promise((resolve) => (runs.size >= count ? resolve() : waiting.push([count, resolve])));
let running = 0;
let most = 0;
const all = runAll(["a", "b", "c", "d"], 2, (item) => {
running += 1;
most = Math.max(most, running);
const done = new Promise((resolve) => runs.set(item, resolve)).then(() => {
running -= 1;
return item.toUpperCase();
});
for (const [count, resolve] of waiting) if (runs.size >= count) resolve();
return done;
});
await started(2);
assert.deepEqual([...runs.keys()], ["a", "b"]); // two at once, c waits for a free slot
runs.get("b")(); // the second ends first: its slot takes c
await started(3);
assert.deepEqual([...runs.keys()], ["a", "b", "c"]);
runs.get("a")();
await started(4);
runs.get("d")();
runs.get("c")();
assert.deepEqual(await all, ["A", "B", "C", "D"]);
assert.equal(most, 2);
assert.deepEqual(await runAll([], 4, async () => 1), []);
});

test("jobCount reads FORK_CI_TEST_JOBS, else the CPU count", () => {
assert.equal(jobCount({ FORK_CI_TEST_JOBS: "3" }), 3);
for (const bad of [undefined, "", "0", "-2", "1.5", "many"])
assert.equal(jobCount({ FORK_CI_TEST_JOBS: bad }), os.availableParallelism(), String(bad));
});

// Runs the real script in a scratch tree whose list names `tests` ({ file: shell text }).
function runScript(tests, jobs) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "fork-ci-tests-"));
const write = (file, text) => {
fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
fs.writeFileSync(path.join(root, file), text);
};
write(
".github/scripts/fork-ci-tests.cjs",
fs.readFileSync(require.resolve("./fork-ci-tests.cjs")),
);
write(".github/fork-ci-tests.txt", Object.keys(tests).join("\n") + "\n");
for (const [file, text] of Object.entries(tests)) write(file, text);
const env = {
...process.env,
FORK_CI_TEST_JOBS: String(jobs),
LEFT: path.join(root, "left.pid"),
MARKS: root,
};
delete env.GITHUB_STEP_SUMMARY;
const run = childProcess.spawnSync("node", [".github/scripts/fork-ci-tests.cjs"], {
cwd: root,
env,
encoding: "utf8",
timeout: 60000,
});
return { root, run };
}

// A test file that passes only when `other` starts while it runs (it waits up to 5 s for the other's mark).
const meets = (name, other) =>
`echo ${name}-start; : > "$MARKS/${name}"\n` +
`for i in $(seq 50); do [ -e "$MARKS/${other}" ] && { echo ${name}-end; exit 0; }; sleep 0.1; done\n` +
`echo "${other} did not run beside ${name}"; exit 1\n`;

test("test files run in parallel, each one's output in its own group, and a failure fails the run", () => {
const { root, run } = runScript(
{
"fork-ci/a.test.sh": meets("a", "b"),
"fork-ci/b.test.sh": meets("b", "a"),
"fork-ci/c.test.sh": "echo c-start; echo c-end; exit 3\n",
},
3,
);
try {
assert.equal(run.status, 1, run.stdout + run.stderr);
for (const name of ["a", "b", "c"])
assert.match(
run.stdout,
new RegExp(
`::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`,
),
);
assert.doesNotMatch(run.stdout, /::error file=fork-ci\/[ab]\.test\.sh/);
assert.match(
run.stdout,
/::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/,
);
} finally {
fs.rmSync(root, { recursive: true, force: true });
}
});

test("FORK_CI_TEST_JOBS=1 runs one file at a time", () => {
const { root, run } = runScript(
{ "fork-ci/a.test.sh": meets("a", "b"), "fork-ci/b.test.sh": meets("b", "a") },
1,
);
try {
assert.equal(run.status, 1, run.stdout + run.stderr);
assert.match(run.stdout, /b did not run beside a/); // b started only after a ended
} finally {
fs.rmSync(root, { recursive: true, force: true });
}
});

test("a process a test leaves behind holding its output does not hang the run", () => {
const { root, run } = runScript(
{ "fork-ci/left.test.sh": 'sleep 60 & echo $! > "$LEFT"; echo done\n' },
1,
);
try {
assert.equal(run.error, undefined, String(run.error)); // not killed at the 60 s limit: the run ended
assert.equal(run.status, 0, run.stdout + run.stderr);
assert.match(run.stdout, /::group::fork-ci\/left\.test\.sh[^\n]*\ndone\n/);
} finally {
try {
process.kill(Number(fs.readFileSync(path.join(root, "left.pid"), "utf8")));
} catch {}
fs.rmSync(root, { recursive: true, force: true });
}
});

// A pipe into a reader that stops early (`grep -q`, `grep -m`, `head`) kills the writer with SIGPIPE
// whenever it has more to write; under `pipefail` that reads as a failed check, more often under load.
const earlyExitPipe =
/(?<!\|)\|(?!\|)\s*!?\s*(?:grep\s+(?:-[A-Za-z]*[qm]|--quiet|--max-count)|head\b)/;
const earlyExitPipes = (text) =>
text
.split("\n")
.map((line, index) => ({ line, number: index + 1 }))
.filter(({ line }) => !/^\s*#/.test(line) && earlyExitPipe.test(line));

test("a pipe into grep -q, grep -m or head is found; reading the whole output first is not", () => {
const found = earlyExitPipes(
[
'sed -n "/a/,$p" f | grep -q x',
"! calls | grep -vqE y",
"git show c | grep -qx z",
"seq 9 | grep -m1 3",
'sed p <<< "$s" | head -n 1',
'grep -q x <<< "$(sed -n p f)"',
"a || grep -q x f",
"# a comment: `x | grep -q` stops early",
'[[ -z "$(grep -E a f | grep -vE b)" ]]',
].join("\n"),
).map(({ number }) => number);
assert.deepEqual(found, [1, 2, 3, 4, 5]);
});

test("no listed shell test pipes into a reader that stops early", () => {
const root = path.resolve(__dirname, "../..");
const shellTests = readList(fs.readFileSync(path.join(root, ".github/fork-ci-tests.txt"), "utf8"))
.map(({ file }) => file)
.filter((file) => file.endsWith(".sh") && fs.existsSync(path.join(root, file)));
assert.ok(shellTests.length > 0);
const found = shellTests.flatMap((file) =>
earlyExitPipes(fs.readFileSync(path.join(root, file), "utf8")).map(
({ line, number }) => `${file}:${number}: ${line.trim()}`,
),
);
assert.deepEqual(found, []);
});
14 changes: 7 additions & 7 deletions .github/scripts/fork-follow.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,7 @@ plan
check "plan: upstream edits a deleted listed file -> the conflict is resolved by deleting it, rebase" \
eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.11 ]] && ! in_candidate ci.yml && [[ ! -e "$root/report.md" ]]'
check "plan: the resolved rebase keeps the fork's patches and its list" \
eval 'action_is rebase && git -C "$root/work" cat-file -e "$(out candidate):feature.txt" && git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt" | grep -qx .github/workflows/ci.yml'
eval 'action_is rebase && git -C "$root/work" cat-file -e "$(out candidate):feature.txt" && grep -qx .github/workflows/ci.yml <<< "$(git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt")"'
# Re-run: an earlier run published this candidate, then stopped before the branch moved. The
# next run rebuilds the same files and advances to that release instead of building again.
released="$(out candidate)"
Expand Down Expand Up @@ -470,7 +470,7 @@ new_fork v0.1.11
fork_commit "fork: list ci.yml" 'echo .github/workflows/ci.yml >> .github/fork-removed-workflows.txt'
plan
check "plan: upstream brings a listed file back -> deleted again on the candidate, rebase" \
eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.12 ]] && ! in_candidate ci.yml && git -C "$root/work" log -1 --format=%s "$(out candidate)" | grep -q "fork: delete upstream workflows listed"'
eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.12 ]] && ! in_candidate ci.yml && grep -q "fork: delete upstream workflows listed" <<< "$(git -C "$root/work" log -1 --format=%s "$(out candidate)")"'

new_upstream
new_fork v0.1.10
Expand Down Expand Up @@ -506,7 +506,7 @@ new_fork v0.1.10
fork_commit "fork: delete ci.yml, CRLF list" "printf '# deleted\\r\\n.github/workflows/ci.yml\\r\\n' > .github/fork-removed-workflows.txt && rm .github/workflows/ci.yml"
plan
check "plan: a list with CRLF line ends is read like one with LF, the listed file stays deleted" \
eval 'action_is rebase && ! in_candidate ci.yml && git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt" | grep -q $'"'"'ci.yml\r$'"'"''
eval 'action_is rebase && ! in_candidate ci.yml && grep -q $'"'"'ci.yml\r$'"'"' <<< "$(git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt")"'

# Upstream renames a listed file: git reports a rename/delete conflict on the new name, which the
# list does not name, so the plan stops and names it rather than shipping it under that name.
Expand Down Expand Up @@ -1054,9 +1054,9 @@ check "workflow: advance's if: is a conjunction (no top-level ||) of the plan's
check "workflow: the move step runs only when the checks before it passed (no if:)" \
eval 'awk "/- name: Move the series branch/ { s = 1; next } s && /^ - / { exit } s && /^ if:/ { bad = 1 } END { exit bad }" <<< "$advance_job"'
check "workflow: action none deletes a leftover candidate branch" \
eval 'grep -B2 -A3 "action == .none." "$workflow" | grep -q "fork-follow-push.sh cleanup"'
eval 'grep -q "fork-follow-push.sh cleanup" <<< "$(grep -B2 -A3 "action == .none." "$workflow")"'
check "workflow: the last sweep reports runs since the run began" \
eval 'sed -n "/^ report:/,\$p" "$workflow" | grep -q "SINCE: \${{ needs.guard.outputs.started }}"'
eval 'grep -q "SINCE: \${{ needs.guard.outputs.started }}" <<< "$(sed -n "/^ report:/,\$p" "$workflow")"'
# Each job that reads FORK_PUSH_TOKEN runs in the Environment that holds it, named as in env.
token_jobs="$(grep -vF "$presence" "$workflow" | awk '/^ [a-z_-]+:$/ { job = $1 } /secrets\.FORK_PUSH_TOKEN/ { print job }' | sort -u)"
env_jobs="$(awk '/^ [a-z_-]+:$/ { job = $1 } /^ environment: fork-push$/ { print job }' "$workflow" | sort -u)"
Expand All @@ -1067,9 +1067,9 @@ check "workflow: the jobs that read FORK_PUSH_TOKEN, and only they, run in the E
check "workflow: no job uses actions/checkout; four plain-git checkouts, none with a token" \
eval '! grep -q "actions/checkout@" "$workflow" && [[ $(grep -c "^ git init -q$" "$workflow") == 4 ]]'
check "workflow: every action is pinned by commit" \
eval '! grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vqE "@[0-9a-f]{40}( |$)"'
eval '[[ -z "$(grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vE "@[0-9a-f]{40}( |$)")" ]]'
check "workflow: the report runs on every run (also action none)" \
eval 'sed -n "/^ report:/,\$p" "$workflow" | grep -q "^ if: always()$"'
eval 'grep -q "^ if: always()$" <<< "$(sed -n "/^ report:/,\$p" "$workflow")"'
check "workflow: upstream workflows are swept after each push" \
eval '[[ $(grep -c "fork-workflows.sh" "$workflow") -ge 3 ]]'

Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/fork-interaction-limit.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,8 @@ expect() { # expect LABEL 'CONDITION' (shell, evaluated)
}
rc_is() { [[ "$rc" == "$1" ]]; }
says() { grep -qF -- "$1" <<< "$out"; }
called() { calls | grep -qF -- "$1"; }
not_called() { ! calls | grep -qF -- "$1"; }
called() { grep -qF -- "$1" <<< "$(calls)"; }
not_called() { ! grep -qF -- "$1" <<< "$(calls)"; }

# --- check -------------------------------------------------------------------------------------
reset; limit collaborators_only 180; run check
Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/fork-run-alert.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ expect() { # expect LABEL 'CONDITION' (shell, evaluated)
}
rc_is() { [[ "$rc" == "$1" ]]; }
says() { grep -qF -- "$1" <<< "$out"; }
called() { calls | grep -qF -- "$1"; }
not_called() { ! calls | grep -qF -- "$1"; }
called() { grep -qF -- "$1" <<< "$(calls)"; }
not_called() { ! grep -qF -- "$1" <<< "$(calls)"; }
issues() { jq length "$root/state/issues.json" 2> /dev/null || echo 0; }

reset; run failure --message "The daily secret sweep failed"
Expand Down
2 changes: 1 addition & 1 deletion .github/scripts/fork-secret-scan.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ elif [[ -n "${GITLEAKS_CONFIG:-}" ]]; then settings="$(cat "$GITLEAKS_CONFIG")"
elif [[ -n "${GITLEAKS_CONFIG_TOML:-}" ]]; then settings="$GITLEAKS_CONFIG_TOML"
elif [[ -f "$source/.gitleaks.toml" ]]; then settings="$(cat "$source/.gitleaks.toml")"
fi
extend_path="$(sed -nE "s/^[[:space:]]*path[[:space:]]*=[[:space:]]*[\"']([^\"']*)[\"'].*/\1/p" <<< "$settings" | head -n 1)"
extend_path="$(sed -nE "s/^[[:space:]]*path[[:space:]]*=[[:space:]]*[\"']([^\"']*)[\"'].*/\1/p" <<< "$settings" | sed -n 1p)"
if [[ "$settings" == *"[extend]"* && -n "$extend_path" && ! -f "$extend_path" ]]; then
echo "FTL Failed to load config error=\"failed to load extended config, err: open $extend_path: no such file or directory\""
exit 1
Expand Down
Loading
Loading