Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/fork-ci-tests.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Every test file under .github/scripts (*.test.*, subdirectories included) and
# Every test file under .github/scripts and fork-ci (*.test.*, subdirectories included) and
# where it runs (.github/scripts/fork-ci-tests.cjs checks this list).
# <test> run by the workflows job, by extension (sh, py, cjs, mjs, js)
# <test> run by <file> run by <file>, which must name the test's path (not one
Expand All @@ -15,6 +15,7 @@
.github/scripts/fork-test-shards.test.cjs
.github/scripts/fork-tests-drift.test.cjs
.github/scripts/fork-workflow-triggers.test.cjs
fork-ci/actions/docs-scope/docs-scope.test.py

# Upstream's tests, run where fork-tests.yml follows upstream's ci.yml.
.github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml
Expand Down
16 changes: 12 additions & 4 deletions .github/scripts/fork-ci-tests.cjs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Fork only: every `*.test.*` under .github/scripts must be listed in .github/fork-ci-tests.txt.
// Fork only: every `*.test.*` under .github/scripts and fork-ci must be listed in .github/fork-ci-tests.txt.
// A plain entry runs here by its extension (RUNNER); `<test> run by <file>` is run by that file,
// which must name the test and cannot be a (disabled) upstream workflow. Unlisted or runner-less
// tests fail; a listed test that is gone fails once the tested history had it, a note before.
Expand All @@ -8,7 +8,7 @@ const path = require("node:path");

const ROOT = path.resolve(__dirname, "../..");
const LIST = ".github/fork-ci-tests.txt";
const SCRIPTS = ".github/scripts";
const TEST_DIRS = [".github/scripts", "fork-ci"];
const TEST_FILE = /\.test\.([^./]+)$/;
const RUNNER = {
sh: ["bash"],
Expand Down Expand Up @@ -83,6 +83,14 @@ function findTests(dir, root = ROOT) {
return out.toSorted();
}

// Every test file under TEST_DIRS. A folder that is not there adds none: a listed test in it is then
// missing, which plan() reports (a failure once the tested history had it).
function presentTests(root = ROOT) {
return TEST_DIRS.filter((dir) => fs.existsSync(path.join(root, dir))).flatMap((dir) =>
findTests(dir, root),
);
}

function git(args) {
return childProcess.execFileSync("git", args, { cwd: ROOT, encoding: "utf8" }).trim();
}
Expand Down Expand Up @@ -113,7 +121,7 @@ function readFile(file) {

function main() {
const entries = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8"));
const result = plan(entries, findTests(SCRIPTS), everInHistory, readFile);
const result = plan(entries, presentTests(), everInHistory, readFile);
const rows = [];
let failed =
result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length;
Expand Down Expand Up @@ -167,6 +175,6 @@ function main() {
if (failed) process.exitCode = 1;
}

module.exports = { plan, readList, findTests };
module.exports = { plan, readList, findTests, presentTests };

if (require.main === module) main();
18 changes: 17 additions & 1 deletion .github/scripts/fork-ci-tests.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ const os = require("node:os");
const path = require("node:path");
const test = require("node:test");

const { plan, readList, findTests } = require("./fork-ci-tests.cjs");
const { plan, readList, findTests, presentTests } = require("./fork-ci-tests.cjs");

const never = () => false;
const here = (...files) => files.map((file) => ({ file, runBy: null }));
Expand Down Expand Up @@ -96,3 +96,19 @@ test("finds every *.test.* file, subdirectories included", () => {
fs.rmSync(root, { recursive: true, force: true });
}
});

test("the composite actions' tests under fork-ci are found too, and a missing folder adds none", () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), "fork-ci-tests-"));
const files = [".github/scripts/a.test.cjs", "fork-ci/actions/x/b.test.py", "other/c.test.py"];
try {
for (const file of files) {
fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
fs.writeFileSync(path.join(root, file), "");
}
assert.deepEqual(presentTests(root), files.slice(0, 2));
fs.rmSync(path.join(root, "fork-ci"), { recursive: true });
assert.deepEqual(presentTests(root), files.slice(0, 1));
} finally {
fs.rmSync(root, { recursive: true, force: true });
}
});
4 changes: 3 additions & 1 deletion .github/workflows/fork-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,12 +176,14 @@ jobs:
timeout-minutes: 5
steps:
# scripts/: some tests are run by a script there (.github/fork-ci-tests.txt).
# fork-ci/: the composite actions and their tests.
- name: Checkout
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
ref: ${{ needs.changes.outputs.sha }}
sparse-checkout: |
/.github/
/fork-ci/
/scripts/
sparse-checkout-cone-mode: false

Expand All @@ -199,7 +201,7 @@ jobs:
tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" actionlint
"$RUNNER_TEMP/actionlint" -color .github/workflows/fork-*.yml

# Every test under .github/scripts, checked against .github/fork-ci-tests.txt:
# Every test under .github/scripts and fork-ci, checked against .github/fork-ci-tests.txt:
# the fork's tests run here, the others must be named by the job or script
# that runs them. An unlisted test, or a listed one deleted from history,
# fails; a listed one whose pull request has not merged yet is a note.
Expand Down
169 changes: 169 additions & 0 deletions fork-ci/actions/docs-scope/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
# docs-scope

A composite action that answers two questions about the change a CI run tests:

- **Is it docs-only?** Then the build and test jobs can pass at once.
- **Which jobs does it reach?** A job runs when a changed file that is not docs matches its paths. A file that is
not docs but that no job names (a root build file, a workflow), or a change to the config, runs every job.

It is the general half of a docs-only gate. The project half (which files are docs, which docs a build or test
reads, which folders each job builds) is a JSON config in the calling repository.

## Usage

Pin the action by full SHA (the tagged one, so a rebase of this fork cannot orphan it):

```yaml
jobs:
build:
steps:
- uses: actions/checkout@<sha> # the default ref: on pull_request GitHub's merge commit ($GITHUB_SHA)
with: { fetch-depth: 0 } # the merge base must be there
# Optional, push only: the last commit whose push run of this workflow passed (see "The base on push").
- id: green
if: github.event_name == 'push'
env: { GH_TOKEN: "${{ github.token }}" } # needs `actions: read`
run: |
sha=$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/ci.yml/runs?branch=main&event=push&status=success&per_page=1" \
--jq '.workflow_runs[0].head_sha // ""') || sha=""
echo "sha=$sha" >> "$GITHUB_OUTPUT"
- id: scope
uses: scuttletoby/t3code/fork-ci/actions/docs-scope@<full sha> # ci/vN
with:
config: .github/docs-scope.json # default
base: ${{ steps.green.outputs.sha }} # empty: a full run
- if: steps.scope.outputs.docs_only != 'true'
run: ./gradlew build
- if: fromJSON(steps.scope.outputs.jobs).engine != 'false'
run: ./gradlew :engine:test
```

Always test `!= 'true'` and `!= 'false'`, never `== 'false'` or `== 'true'`: a missing or empty output then means
"run". `jobs` is `{}` when the change or the config is unknown, so every job reads as "run".

Run it as a step of the jobs it gates, as above. If it runs in a job of its own that others `need`, a failure of that
job skips them, and GitHub counts a skipped required job as passing: gate them with
`if: ${{ !cancelled() && (needs.scope.result != 'success' || fromJSON(needs.scope.outputs.jobs).engine != 'false') }}`.

### Outputs

| Output | Value |
| ----------- | ------------------------------------------------------------------------------ |
| `docs_only` | `true` only when every changed file is docs and the event may be docs-only |
| `jobs` | JSON `{"<job>": "true" \| "false"}`; `{}` when unknown |
| `reason` | one line: `docs-only:`, `full run:`, `scoped run:` or `not docs-only:` and why |

The step log also prints one line per job: `job engine: run (engine/A.java matches its paths)` or
`job engine: skip (...)`. Control characters, line separators and bytes that are not UTF-8 in a file name are
printed escaped (`\n`, `\udcff`), so a file name can neither start a `::workflow-command::` line nor add an output.

## What "the change" is

- **pull_request:** the union of two diffs without rename detection (a rename is its old and its new path):
merge base .. PR head (every push of the PR together, never the last push alone) and base .. GitHub's merge commit.
HEAD must be that merge commit: HEAD == `$GITHUB_SHA`, it has two parents, and HEAD^2 is the event's
`pull_request.head.sha`. Only then is HEAD^1 trusted as the base. A checkout of the PR head is a full run: if its
last commit merged main in, HEAD^1 would be the PR's own commit, the PR's code would drop out of both diffs and the
config would be read from the PR.
- **Any other event** (push, merge_group, workflow_dispatch): `base` .. HEAD, only when the caller gives `base` and
HEAD == `$GITHUB_SHA`, and the base must be on HEAD's first-parent line. Without a base the run is a full run. Such
an event is docs-only only when `docs_only_events` names it; its `jobs` are read either way, so a docs-only push to
main can still skip the build jobs.

### The base on push

Pass the last commit whose **push** run of the workflow on main **passed**, or nothing (a full run). Never
`github.event.before`: if push A (code) went red or was cancelled (for example by `concurrency`), push B (docs) diffs
B against A, every job reads `false`, and main turns green with A's code never tested; a rerun of B gives the same
answer. Against the last green commit, A's code is in the diff until a run that tests it passes.

Keep `event=push` in the query: `branch` matches a run's head branch, so without it a passing `pull_request` run from
a fork whose branch is also called `main` counts, and its head commit is a PR head that no run tested as main (the PR
run tested GitHub's merge commit). The action enforces the same rule: the base must be on HEAD's first-parent line
(main's own pushes), so a merged branch's commit, another branch or a force-pushed-away commit is a full run, and so
is a base that is not in the checkout. A rerun can see a newer green base than the first run did; that only shrinks
the diff to commits a passing push run already tested.

## Config

```json
{
"docs": ["**/*.md"],
"never_docs": [
{ "glob": "**/src/**", "why": "inside a source set (packed into jars)" },
{ "glob": "**/tests/**", "why": "inside a test folder" },
{ "glob": "tools/CLAIMS.md", "why": "read by the release tool" }
],
"never_docs_from": [
{
"file": "tools/modules.json",
"list": "modules",
"field": "inputs",
"label": "id",
"why": "a module input"
}
],
"docs_only_events": ["pull_request"],
"jobs": {
"engine": {
"paths": ["engine/**", "shared/items/**"],
"paths_from": [
{
"file": "tools/modules.json",
"list": "modules",
"where": { "kind": "engine-module" },
"field": ["paths", "inputs"],
"label": "id"
}
]
}
}
}
```

- `docs` (required): globs of docs files. `**/` is any number of folders, `**` anything (a newline in a name too),
`*` and `?` stay in one name. A name with a control character or bytes that are not UTF-8 is never docs.
- `never_docs`: files that match `docs` but that a build, test or tool reads; each says why.
- `never_docs_from` / `paths_from`: paths listed in a JSON file of the repository. `list` is a dotted key to a list
of objects (none: the file itself), `where` keeps the objects whose keys equal these values, `field` names the key
(or keys) holding a path or a list of paths, `label` names an object in the log. An entry matches a path exactly or
as a folder. The file is read at the base and at the head; an entry at either makes a file not docs or runs a job,
so a pull request cannot free a file by removing it from the list. Whether a file is named by no job (so every job
runs) is judged by the base's entries only: a pull request that adds an entry for a root file, even with a new
module, adds that job but cannot stop the full run. A list that could drop a rule quietly is an error: a kept
object with none of the `field` keys, a `field` key no kept object has (a typo, or a `where` that keeps nothing),
or an entry that is not a plain repository path (`./x`, `/x`, `x/../y`, `x//y`, a wildcard), which could never
match.
- `docs_only_events`: events that may be docs-only (default `["pull_request"]`).
- `jobs`: per job, `paths` globs and `paths_from` lists. Docs files never run a job; a `never_docs` file does. A
file that is not docs and that no job names runs every job, and so does a change to the config.

Unknown keys and wrong types are an error (a typo such as `never_doc` must not drop a rule quietly), and an error
means a full run that names it. The `config` input must be a plain repository path (no `./`, `..` or leading `/`),
so the "config is never docs" rule cannot be dodged by spelling.

## Why a pull request cannot loosen it

- The script runs from the action's own folder, at the SHA the caller pins: never a copy from the checkout.
- The config is read from the base commit with `git show`, never from the working tree. A pull request that changes
the config is a full run (the config file is never docs).

What it cannot stop: a `pull_request` run uses the workflow file from the pull request, so a pull request that edits
the `uses:` line chooses the code that judges it. That edit is a change to a non-docs file, so every correct version
runs in full for it; pin only SHAs that this fork's CI tested and tagged.

## Fails closed

A full run (and `jobs` = `{}`) for: an event without a base, a base off HEAD's first-parent line, HEAD that is not
`$GITHUB_SHA`, a pull_request HEAD that is not GitHub's merge commit (two parents, HEAD^2 = the event's PR head; no
event file), no merge base (shallow checkout), a base that is not a commit, a git error, an empty change, a config
path that is not plain, no config at the base, a config that does not parse or has unknown keys or wrong types, a
`*_from` file at neither side or not readable as configured (not UTF-8, nested too deep, a missing field, an entry
that is not a plain path). Any other error is caught too and gives a full run with its message, never a step without
outputs. A symlink or submodule is never docs, and the diff uses `--ignore-submodules=none`, so neither
`ignore = all` in `.gitmodules` nor the runner's `diff.ignoreSubmodules` can hide a gitlink bump.

## Tests

`docs-scope.test.py` builds real git repositories with GitHub-style merge commits. It runs in the fork's CI
(`.github/fork-ci-tests.txt`): `python3 fork-ci/actions/docs-scope/docs-scope.test.py`.
42 changes: 42 additions & 0 deletions fork-ci/actions/docs-scope/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: docs-scope
description: >-
Is this change docs-only, and which configured jobs does it reach? Reads its rules from the base
commit and runs its own code at the pinned SHA, so a pull request cannot loosen what judges it.
Anything unclear means a full run. Usage: README.md next to this file.
inputs:
config:
description: >-
The JSON config, a plain path in the repository (no ./); read at the base commit, never from the checkout.
required: false
default: .github/docs-scope.json
base:
description: >-
Events other than pull_request only: the commit to compare HEAD with, the last commit whose
push run passed (event=push; never github.event.before: see README), on HEAD's first-parent
line. Empty or off that line: such runs run in full.
required: false
default: ""
outputs:
docs_only:
description: "'true' only when every changed file is docs and the event may be docs-only; test != 'true'."
value: ${{ steps.scope.outputs.docs_only }}
jobs:
description: >-
JSON {"<job>": "true"|"false"}; {} when the change or the config is unknown. Read a job as
fromJSON(steps.<id>.outputs.jobs).<job> != 'false', so a missing job runs.
value: ${{ steps.scope.outputs.jobs }}
reason:
description: One line saying why.
value: ${{ steps.scope.outputs.reason }}
runs:
using: composite
steps:
# Only code from this action's folder (the caller's pinned SHA) runs; nothing from the checkout.
# The script also reads the runner's GITHUB_SHA and GITHUB_EVENT_PATH (HEAD must be the run's commit).
- id: scope
shell: bash
env:
CONFIG: ${{ inputs.config }}
BASE: ${{ inputs.base }}
EVENT: ${{ github.event_name }}
run: python3 -I "$GITHUB_ACTION_PATH/docs_scope.py" --config "$CONFIG" --event "$EVENT" --base "$BASE"
Loading
Loading