diff --git a/.gitea/workflows/nightly.yml b/.gitea/workflows/nightly.yml index 4ec3cf2..6989264 100644 --- a/.gitea/workflows/nightly.yml +++ b/.gitea/workflows/nightly.yml @@ -2,34 +2,40 @@ # mojo-status-strict (`pixi run nightly`). A failure opens nothing and sends nothing: the run goes red where anyone # looking at the repository's Actions page sees it. docs/ci.md says how to reproduce a seed and what to do with it. # -# Gitea Actions (host mode) as test.yml; on GitHub Actions the same file runs with `runs-on: ubuntu-latest`. +# Gitea Actions, on the `heavy` runner test.yml describes (never the deploy server); on GitHub Actions the same file +# runs with `runs-on: ubuntu-latest`. name: Nightly on: schedule: - cron: "17 6 * * *" # 06:17 UTC, before the working day in America workflow_dispatch: +# one nightly run at a time: while no `heavy` runner is registered a run waits, and the next night's replaces it +concurrency: + group: nightly + cancel-in-progress: true + env: CI: "true" PIXI_VERSION: "0.83.0" jobs: fuzz: - runs-on: quarto-deploy + runs-on: heavy timeout-minutes: 90 steps: - uses: actions/checkout@v4 with: submodules: recursive - # pixi: the host's when it is the pinned version, else the pinned release, fetched once into a cache directory - # the runner's user owns (no sudo), checked by SHA-256 + # pixi: the image's when it is the pinned version, else the pinned release, fetched into a cache directory + # (which lasts only where the runner keeps a home between jobs) and checked by SHA-256 - name: pixi, pinned shell: bash run: | set -euo pipefail if command -v pixi >/dev/null && [ "$(pixi --version | cut -d' ' -f2)" = "$PIXI_VERSION" ]; then - echo "pixi $PIXI_VERSION is on the host: $(command -v pixi)"; exit 0 + echo "pixi $PIXI_VERSION is on the PATH: $(command -v pixi)"; exit 0 fi dir="${PPLUS_CI_CACHE:-$HOME/.cache/pplus-ci}/pixi-$PIXI_VERSION" if [ ! -x "$dir/pixi" ]; then diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 970fe34..07edb28 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2,8 +2,11 @@ # environment pixi.lock pins (Node, Python, the Mojo compiler), so CI and a developer cannot drift. docs/ci.md says # what it runs, how long it takes, and what to do when it fails. # -# Gitea Actions, on a runner in host mode (no container: steps run on the host's shell; pplus-web's docs/deploy.md -# has the runner's setup). The same file runs on GitHub Actions with one change: `runs-on: ubuntu-latest`. +# Gitea Actions, on the runner with the label `heavy`: a machine with cores to spare, not the server that deploys +# the site, because the gate builds and runs some 300 Mojo programs and fuzzes (docs/ci.md has the runner's setup). +# With no such runner registered a run waits; nothing here runs on the deploy server. What the job needs beyond git, +# Node and curl comes from pixi.lock or is fetched by a pinned hash, so the runner may be a container or a host. +# The same file runs on GitHub Actions with one change: `runs-on: ubuntu-latest`. name: Test on: push: @@ -17,15 +20,15 @@ concurrency: env: # set by Gitea and GitHub Actions already; written here so the tests' "fail, not skip, under CI" never depends on it CI: "true" - # the pixi that reads pixi.lock; when the host has another version, this one is fetched into the cache below + # the pixi that reads pixi.lock; when the runner has another version, this one is fetched into the cache below PIXI_VERSION: "0.83.0" jobs: gate: - # a pull request runs only from a branch of this repository: on a host runner a fork's workflow would run its - # own code on the droplet (docs/deploy.md in pplus-web). Drop this line for a runner that is a throwaway VM. + # a pull request runs only from a branch of this repository: a fork's workflow would run its own code on the + # runner (docs/ci.md). Drop this line for a runner that is a throwaway machine. if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: quarto-deploy + runs-on: heavy timeout-minutes: 60 steps: # library/ is pplus-library, a submodule pinned by commit: the programs, the corpus, the routines @@ -33,14 +36,14 @@ jobs: with: submodules: recursive - # pixi: the host's when it is the pinned version, else the pinned release, fetched once into a cache directory - # the runner's user owns (no sudo), checked by SHA-256 + # pixi: the image's when it is the pinned version, else the pinned release, fetched into a cache directory + # (which lasts only where the runner keeps a home between jobs) and checked by SHA-256 - name: pixi, pinned shell: bash run: | set -euo pipefail if command -v pixi >/dev/null && [ "$(pixi --version | cut -d' ' -f2)" = "$PIXI_VERSION" ]; then - echo "pixi $PIXI_VERSION is on the host: $(command -v pixi)"; exit 0 + echo "pixi $PIXI_VERSION is on the PATH: $(command -v pixi)"; exit 0 fi dir="${PPLUS_CI_CACHE:-$HOME/.cache/pplus-ci}/pixi-$PIXI_VERSION" if [ ! -x "$dir/pixi" ]; then @@ -60,8 +63,8 @@ jobs: - name: The pinned environment, exactly as pixi.lock has it run: pixi install --locked - # .tools/ holds the pinned Dafny and mojo-local's answers (keyed by compiler, arguments and source); on a host - # runner it lives in the cache directory, so neither is fetched or built again on every push + # .tools/ holds the pinned Dafny and mojo-local's answers (keyed by compiler, arguments and source); it is a + # link into the cache directory, so where the runner keeps a home between jobs neither is fetched or built again - name: The pinned Dafny shell: bash run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e06264..6ea4b89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Mojo 1.2.0, held to the real compiler on the machine as well as on godbolt.org, - Mojo is built one stated way everywhere core builds it or asks godbolt to (the Compile button, the corpus, the library's references, `mojo-local`, `mojo-fuzz`, `mojo-run`): the arguments are `build.args` in the library's pin, before a program's own. `--target-cpu x86-64-v3` makes a listing the same whichever of godbolt's machines answers (one answered with AVX-512, which the model machine does not know); `--fp-mode contract=off` rounds every Float64 operation on its own, as both interpreters do, where Mojo's default fuses a multiply with an add. `BUILD_ARGS` carries them and `BUILD_LEVELS` is new. `mojo build` with no arguments targets the machine it runs on and may fuse (`docs/trust.md`). - The Mojo syntax definition the pipeline writes states its license as Apache-2.0 (it said MIT), and `THIRD_PARTY.md` says that the engines and the lessons, which came from the course, are under this repository's copyright and license. +- `pixi run cli prove` has no time limit unless `--budget-ms` gives one, and `library-check` proves without one: their answers are the prover's counted budgets' alone, the same on every machine. The clock is the caller's (`provePseudo(parsed, { budgetMs })`, `Workspace.proofBudgetMs`); left out, it is the Prove tab's 3 seconds (`PROVE_BUDGET_MS`), as before. +- The pinned environment has a C compiler on Linux (`gcc` from conda-forge, with the C library it links against pinned to glibc 2.34, which Mojo's runtime needs): `mojo build` links with the C compiler it finds, so until now a build depended on the system's, and on a machine without one (CI's image) no Mojo program could be built. The C references and the C harness are built with it too. `mojo-local` reports a program that could not be linked as that, not as rejected by the compiler, and does not keep the answer. + ### Added - `pixi run mojo-local` builds every Mojo program core holds or generates on the pinned local compiler and holds the checker, the interpreter and the translator to it; `pixi run mojo-fuzz` does the same with generated programs. @@ -30,6 +33,8 @@ Mojo 1.2.0, held to the real compiler on the machine as well as on godbolt.org, ### Fixed +- The gate no longer depends on the machine it runs on. It first ran on the server that deploys the site, a slow container image without a C compiler or ICU, where ten tests failed on correct code and no Mojo program could be built. The pinned Dafny starts without ICU (it is run with `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1`), and `dafny-fetch` fails when the Dafny it fetched does not start (it printed an empty version and went on). A test's time limit is processor time at the machine's pace (`tests/helpers/pace.mjs`), and the tests that hold what the prover proves give it no time on the clock. +- The workflows run on a runner with the label `heavy`, not on the deploy server: the gate builds and runs some 300 Mojo programs. `docs/ci.md` says how to register one, and describes the runners as they are (the workflows' comments described a host-mode runner that was never what ran them). - The translator writes Mojo the compiler accepts and that prints what the pseudo prints. - The Mojo interpreter: a numeric literal beside a Float32 is a Float32; a float to a whole power is computed as the compiler computes it, and to any other power the interpreter declines; a list in brackets takes a List's element type. - The prover: 18 rules that proved what a run breaks are closed, each with a regression program in the library. diff --git a/README.md b/README.md index 45fddc5..1992e06 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ What core holds: - verified routines (statistics, numerics, alignment, molecular simulation, sequences, special functions) that a program brings in with `USE`, each checked by every tool `pixi run library-check` has ([docs/library.md](docs/library.md)); - the Mojo corpus (`library/mojo/`): the pin, material harvested from Modular's repositories at pinned commits, our own probes and programs, and godbolt's stored answer for every program we wrote and every harvested program that runs (the compiler's own diagnostic tests are asked only on request) ([docs/mojo-corpus.md](docs/mojo-corpus.md)); - pseudo regression programs and the prover's soundness regressions (`library/pseudo/`). -- **A command line:** `pixi run cli check|run|prove|fuzz|translate ` (`translate` takes `--to mojo`, `lingo`, `matho` or `speco`). +- **A command line:** `pixi run cli check|run|prove|fuzz|translate ` (`translate` takes `--to mojo`, `lingo`, `matho` or `speco`; `prove` has no time limit unless `--budget-ms` gives one, so it says the same on every machine). What core says about Mojo is pinned to **Mojo 1.2.0**. The checker and the interpreter are held to the real compiler's answers on the corpus: to godbolt's stored answers in the tests, and to the pinned compiler on a developer's machine with `pixi run mojo-local` and `mojo-fuzz`, which also build the Mojo the translator writes. On a program outside the corpus the interpreter can still run something the compiler rejects; [docs/trust.md](docs/trust.md) says what each verdict means and what it does not. diff --git a/THIRD_PARTY.md b/THIRD_PARTY.md index a727b5f..be85397 100644 --- a/THIRD_PARTY.md +++ b/THIRD_PARTY.md @@ -40,6 +40,7 @@ pixi installs these into `.pixi/`, npm into `node_modules/`, and `scripts/dafny. | Mojo (`mojo`, `mojo-compiler`, `mojo-python`, `mblack`) | 1.2.0 | conda.modular.com, by pixi | LicenseRef-Modular-Proprietary (Modular's license, as `pixi.lock` records it) | Nothing. Tasks run the compiler on our programs. | | Node.js | 26.10.0 | conda-forge, by pixi | MIT | Nothing. | | Python | 3.14.7 | conda-forge, by pixi | Python-2.0 | Nothing. The pipelines use its standard library. | +| GCC (`gcc`, on Linux only) | 16.2.0 | conda-forge, by pixi | GPL-3.0-only WITH GCC-exception-3.1 (the compiler, `gcc_impl_linux-64`), with binutils (GPL-3.0-only) and the headers and C library it builds against (LGPL-2.0-or-later) | Nothing. It links the programs `mojo build` makes and builds the C references `library-check` runs and one test's harness; the programs are ours, and no binary is kept. | | TypeScript | 6.0.3 | npm (`package-lock.json`) | Apache-2.0 | Nothing. `tsc` compiles `src/`; the output is our code. | | Dafny | 4.11.0 | the GitHub release zip, by `scripts/dafny.py`, checked by SHA-256 | MIT (Dafny and the Z3 it carries, with .NET runtime files; Dafny's own NOTICES.txt lists the rest) | Nothing. Its verdicts are stored in `library/modules/*/*/golden/dafny.json`. | diff --git a/docs/ci.md b/docs/ci.md index d5cb9f3..1df0373 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,24 +1,38 @@ # CI -CI runs what a developer runs. Every workflow installs pixi, restores the environment `pixi.lock` pins (Node, Python, the Mojo compiler) with `pixi install --locked`, and runs a pixi task, so the gate on the runner and the gate on a laptop are the same command in the same environment. Five weeks once passed with `main` broken in two places and nobody told: CI then ran `npm ci && npm test` with whatever Node the host had, no pandoc, no Mojo compiler and no Dafny, and a missing tool skipped tests in silence. +CI runs what a developer runs. Every workflow installs pixi, restores the environment `pixi.lock` pins (Node, Python, the Mojo compiler, and a C compiler on Linux) with `pixi install --locked`, and runs a pixi task, so the gate on the runner and the gate on a laptop are the same command in the same environment. Five weeks once passed with `main` broken in two places and nobody told: CI then ran `npm ci && npm test` with whatever Node the host had, no pandoc, no Mojo compiler and no Dafny, and a missing tool skipped tests in silence. | Workflow | When | Runs | Time here | |---|---|---|---| -| `.gitea/workflows/test.yml` | every push and pull request | `pixi run ci`, then checks that nothing the gate regenerates differs from what is committed | 4 minutes here, 2.5 with `mojo-local`'s answers cached | -| `.gitea/workflows/nightly.yml` | every day at 06:17 UTC, and by hand | `pixi run nightly` | 12 minutes here | -| pplus-library's `.gitea/workflows/test.yml` | every push and pull request to the library | core's default branch with that library commit in `library/`, then `pixi run ci` (below) | as core's | +| `.gitea/workflows/test.yml` | every push and pull request, on the `heavy` runner | `pixi run ci`, then checks that nothing the gate regenerates differs from what is committed | 4 minutes here, 2.5 with `mojo-local`'s answers cached | +| `.gitea/workflows/nightly.yml` | every day at 06:17 UTC, and by hand, on the `heavy` runner | `pixi run nightly` | 12 minutes here | +| pplus-library's `.gitea/workflows/test.yml` | every push and pull request to the library, on the `heavy` runner | core's default branch with that library commit in `library/`, then `pixi run ci` (below) | as core's | `pixi run ci` is `test` (every test), `mojo-status-strict` (no Mojo program of ours fails, from godbolt's stored answers), `library-check` (every library routine at every level, Dafny included), `mojo-local` (every Mojo program core holds or generates, built and run by the pinned compiler) and `ci-fuzz` (below). `pixi run nightly` is `mojo-status-strict` and the fuzzers for longer. ## The runner -The workflows are written for Gitea Actions on a runner in host mode (no container: steps run on the host's shell); pplus-web's `docs/deploy.md` has the setup. They use no `container:` and no `services:`, and each needs one change to run on GitHub Actions: `runs-on: ubuntu-latest` in place of the host's label. +The gate is heavy: it builds and runs some 300 Mojo programs, checks every library routine and fuzzes, about 4 minutes on a 24-core workstation, and 4 and a half on two cores in the deploy runner's image with nothing cached. So the workflows run on a runner with the label `heavy`, a machine with cores to spare, and never on the server that deploys the site (pplus-web's deploy job, label `quarto-deploy`, runs the site's own tests and nothing of this gate). While no `heavy` runner is registered a run waits, and the next push's run replaces it; until one is, the gate is `pixi run ci` on a developer's machine before a push. -- **pixi.** `PIXI_VERSION` at the top of each workflow is the pixi that reads the lock. The first step uses the host's pixi when it is that version; otherwise it downloads that release from GitHub into `$PPLUS_CI_CACHE/pixi-/` (default `~/.cache/pplus-ci`), checks its SHA-256 (written in the workflow), and puts it on the PATH. No sudo; a host runner fetches it once. To move pixi: change `PIXI_VERSION` and both hashes in every workflow, in one commit, and check that `pixi install --locked` still accepts the lock. -- **The environment.** `pixi install --locked` fails if `pixi.toml` and `pixi.lock` disagree, so the runner never resolves anything new. pixi's package cache stays on the host, so a host runner downloads each package once. -- **`.tools/`.** The pinned Dafny (`pixi run dafny-fetch`, about 150 MB) and `mojo-local`'s answers live in `.tools/`; in CI `.tools` is a link to `$PPLUS_CI_CACHE/core-tools`, so neither is fetched or rebuilt on every push. `mojo-local` keys an answer by the compiler's version, the arguments, the input and the whole source, so a cached answer is the compiler's answer for exactly that program. Two jobs that share the cache at once could write the same answer file together; the runner takes one job at a time unless its `capacity` is raised. -- **`CI`.** Each workflow sets `CI=true` (Gitea and GitHub set it too). Under it a test that would skip for a missing tool fails instead, because a skip in CI is how a gap hides: python3 (`tests/mojo.test.mjs`, `tests/dafny.test.mjs`, `tests/library.test.mjs`), the fetched Dafny (`tests/dafny.test.mjs`), a git checkout to read the library pointer from (`tests/chain.test.mjs`), and in pplus-web pandoc for the Quarto tests. -- **What the host needs besides pixi:** git, curl, tar, `sha256sum`, bash, and Node for `actions/checkout` itself; a C compiler (`cc`) for the C references `library-check` builds. Without `cc` those checks are recorded as skipped, `library/status.json` then differs from the committed one, and the last step of `test.yml` fails and says so. +Registering one, on the machine that will run it (a Gitea act_runner, as its documentation describes: download it, `act_runner generate-config > config.yaml`, run it as a service under a user of its own): + +```bash +# in a container, the image the deploy runner uses (anything with git, Node, curl, tar and bash will do): +act_runner register --instance https:// --token --no-interactive \ + --labels "heavy:docker:///infra/quarto-deploy:1.9.38" +# or on the machine's own shell (host mode), where the caches below last from job to job: +act_runner register --instance https:// --token --no-interactive --labels "heavy:host" +``` + +Take the token from the organization's runner settings, so that pplus-core and pplus-library both see the runner. In `config.yaml`, `runner.capacity` is the number of jobs at once: leave it at 1 unless each job has a cache directory of its own (`.tools/`, below). The workflows use no `container:` and no `services:` of their own, and each needs one change to run on GitHub Actions: `runs-on: ubuntu-latest` in place of the label. A pull request from a fork would run its own code on that machine; each job's `if:` runs pull requests only from branches of the repository itself. + +- **pixi.** `PIXI_VERSION` at the top of each workflow is the pixi that reads the lock. The first step uses the runner's pixi when it is that version; otherwise it downloads that release from GitHub into `$PPLUS_CI_CACHE/pixi-/` (default `~/.cache/pplus-ci`), checks its SHA-256 (written in the workflow), and puts it on the PATH. No sudo. A runner that keeps its home between jobs (host mode) fetches it once; a container fetches it in every job. To move pixi: change `PIXI_VERSION` and both hashes in every workflow, in one commit, and check that `pixi install --locked` still accepts the lock. +- **The environment.** `pixi install --locked` fails if `pixi.toml` and `pixi.lock` disagree, so the runner never resolves anything new. On Linux the environment has a C compiler (`gcc` from conda-forge, which is also `cc` on the PATH), because `mojo build` links every program with the C compiler it finds and stops with "unable to find suitable c compiler for linking" when there is none, as in a slim container image; `library-check` builds the C references with it and one test a C harness. The C library it links against is pinned too (`sysroot_linux-64` 2.34): Mojo 1.2.0's runtime libraries need glibc 2.34, and against conda-forge's default, 2.28, no Mojo program links. So under `pixi run` it is this compiler that links, on a workstation as on the runner, whatever the system has. Where pixi's package cache lasts, each package is downloaded once; a container downloads the whole environment in every job. +- **`.tools/`.** The pinned Dafny (`pixi run dafny-fetch`, about 150 MB) and `mojo-local`'s answers live in `.tools/`; in CI `.tools` is a link to `$PPLUS_CI_CACHE/core-tools`, so where that directory lasts neither is fetched or rebuilt on every push. `mojo-local` keys an answer by the compiler's version, the arguments, the input and the whole source, so a cached answer is the compiler's answer for exactly that program. Two jobs that share the cache at once could write the same answer file together; the runner takes one job at a time unless its `capacity` is raised. +- **Dafny without ICU.** The release zip carries its own .NET runtime but not ICU, and .NET aborts without it unless told not to look: in the deploy runner's image, which has none, the fetched Dafny printed no version and aborted. `scripts/dafny.py` (and `scripts/library.mjs` and `tests/dafny.test.mjs`, which run the executable themselves) set `DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1`, and `dafny-fetch` fails if the Dafny it fetched does not start. +- **`CI`.** Each workflow sets `CI=true` (Gitea and GitHub set it too). Under it a test that would skip for a missing tool fails instead, because a skip in CI is how a gap hides: python3 (`tests/mojo.test.mjs`, `tests/dafny.test.mjs`, `tests/library.test.mjs`), the fetched Dafny (`tests/dafny.test.mjs`), `cc` (`tests/library-harness.test.mjs`), a git checkout to read the library pointer from (`tests/chain.test.mjs`), and in pplus-web pandoc for the Quarto tests. +- **What the runner needs besides pixi:** git, curl, tar, `sha256sum`, bash, and Node for `actions/checkout` itself. Everything else is pinned here: the compilers by `pixi.lock`, Dafny by its hash. +- **A slow or busy machine changes how long the gate takes, not what it finds.** The gate first ran on the deploy server, a small virtual machine: the tests that take 20 seconds on a workstation took 15 to 22 minutes there, work that takes 5 milliseconds took more than 3 seconds, and ten tests failed on correct code. Two things in the tests depended on speed then and do not now. A time limit in a test ("this reads in under 2 seconds", there to catch work that grows with the square of its input) is processor time at the machine's pace: `tests/helpers/pace.mjs` times a fixed piece of work and multiplies the limit by how much longer it takes than on the machine the limits were written on, never by less than 1. And the prover's time on the clock, 3 seconds for each routine in the Prove tab, is the caller's: the tests, `pixi run cli prove` and `library-check` give it none, so the counted budgets decide alone and the answer is the same everywhere ([limits.md](limits.md#proving-and-testing)). The fuzzers keep their clock (`--budget-ms`, below): on a slower machine `prove-fuzz` proves less, never more. ## The fuzzers' seeds diff --git a/docs/engines.md b/docs/engines.md index 6496e46..4b72de4 100644 --- a/docs/engines.md +++ b/docs/engines.md @@ -46,7 +46,7 @@ The reference's "Limits" and "What stops a run" tables (`src/common/reference.ts - **What the parser reads.** A line inside more than 200 blocks, or brackets, calls, unary operators or types nested more than 200 deep, is one `too-deep` error (`MAX_NESTING` in `parse.ts`); so is a chain of more than 400 operands (`1 + 1 + ...`, `MAX_CHAIN`). The two are counted apart and a chain leans one level per operand, so brackets that each hold a long chain would multiply them; an expression whose tree is taller than 400 levels is therefore `too-deep` as well (`MAX_TREE`, measured where each expression is complete). No later stage (checker, analysis, translators) walks a taller tree; `tests/pseudo-robustness.test.mjs` takes the tallest through every one of them. - **What a run holds.** One collection holds at most 2,000,000 values (an ARRAY's declared size, `FILL`, a range walked by `FOR EACH` or listed for a quantifier), and a text at most 10,000,000 characters (`MAX_VALUES`, `MAX_TEXT` in `interp.ts`); past either the run stops (kind `value`) instead of allocating. - **How long a run goes.** The step limit (20,000 in the app's runs, `maxSteps`) stops a loop that may never end (kind `limit`). The positions FORALL, EXISTS, SUM and sets written by rule visit are counted apart, up to 100 per step allowed, so one quantifier inside another cannot run without end. Calls go 400 deep (`maxDepth`, kind `depth`); a recursion whose call sits inside many nested blocks can exhaust the engine underneath before 400 calls (with ten blocks around the call, somewhere between 200 and 300 calls, varying from run to run), which is the same stop, at the line the run was on. -- **The Prove tab's budget.** The verifier's work is counted, not timed (`work` in `logic.ts`): one obligation's proof, one subroutine's paths and invariants, its proofs, and the whole program each have a budget, the same on every machine, and past one the answer is open ("too large to prove here") or the subroutine partial, never proved ([limits.md](limits.md#proving-and-testing)). A wall-clock budget stays behind them as a last resort: `provePseudo` gives the verifier 3 seconds per unit, then tries generated inputs on each routine within a counted budget of steps and also for up to 3 seconds (`checkProperties` with `maxWork` and `budgetMs`): a slow routine is tried on fewer inputs, not waited for. +- **The Prove tab's budget.** The verifier's work is counted, not timed (`work` in `logic.ts`): one obligation's proof, one subroutine's paths and invariants, its proofs, and the whole program each have a budget, the same on every machine, and past one the answer is open ("too large to prove here") or the subroutine partial, never proved ([limits.md](limits.md#proving-and-testing)). A wall-clock budget stays behind them as a last resort: `provePseudo` gives the verifier 3 seconds per unit, then tries generated inputs on each routine within a counted budget of steps and also for up to 3 seconds (`checkProperties` with `maxWork` and `budgetMs`): a slow routine is tried on fewer inputs, not waited for. The 3 seconds are the Prove tab's (`PROVE_BUDGET_MS`); a caller gives another with `provePseudo(parsed, { budgetMs })` or `Workspace.proofBudgetMs`, and the command line, `library-check` and the tests give none, so their answers do not depend on the machine ([limits.md](limits.md#proving-and-testing)). - **Recording.** A recorded run (the stepper, the table of values) keeps a picture of the variables after every step, and a picture copies each variable the step changed. A large collection that changes at many steps is copied at each of them, so once the copies of a run come to more than 10,000,000 values (about 80 MB of numbers; `maxRecorded` in `interp.ts`), the trace ends with a step of kind `unrecorded` that says so, and the run goes on without recording: its output, result, stop and line counts are the whole run's. No program of the corpus, the exercises or the lessons comes near it (the largest copies about 42,000). `tests/pseudo-robustness.test.mjs` holds the cut with a small budget. - **Checker errors and stops added in the 2026-11 audits** (each in `tests/pseudo-robustness.test.mjs`, `tests/numeric.test.mjs` or `tests/reference.test.mjs`): `too-deep` (the limits above), `literal-range` for a whole-number literal past 2^53 − 1 and a FLOAT or FLOAT32 literal past its largest value, `recursive-type`, `array-of-array`, `array-length` (an ARRAY given a LIST or ARRAY of another length, when both are written as numbers), `bad-member` (a SET of anything but single values). Stops: the collection and text caps, the quantifier positions, an ARRAY given a value of another length at a store, a call or a RETURN, a FOR counter declared with a sized type leaving its type, a partial sum or product of `SUM_OF` or `DOT` past 2^53 − 1, `CHR` of a surrogate half, `TO_FLOAT` or `INPUT` of text that is not a decimal number, and NaN or an infinity passed to a FLOAT parameter from outside without `USE IEEE`. diff --git a/docs/limits.md b/docs/limits.md index 5410d5a..5614300 100644 --- a/docs/limits.md +++ b/docs/limits.md @@ -73,7 +73,7 @@ The prover's work is counted, not timed: each term it builds or visits, each cas | The routines of one program the fuzzer tests | `FUZZ_ROUTINES`, `src/common/workspace.ts` | the first 100 | A report with no cases: "Not tested: generated inputs cover the first 100 routines of a program, and this is routine N." | | The fuzzer's work for a whole program | `FUZZ_PROGRAM_INPUTS`, `src/common/workspace.ts` | 600 inputs' worth: 1,800,000 at the default budget | A report with no cases: "Not tested: the routines before this one took all the work the tests of one program may take (1,800,000 steps)." | -The wall-clock budget the Prove tab gives each subroutine (3 seconds) stays behind these as a last resort. Measured on the machine this was written on: 20 nested `ABS` (or a matho pane `f(x) = ||||...x...||||` of 20 bars) took a 1 GB heap down before, and now ends in 0.1 seconds in a 128 MB heap; a subroutine at `UNIT_WORK` takes 0.7 to 2 seconds, one at `PROOF_WORK` about 1 second, and a program at `PROGRAM_WORK` about 4 seconds, each in a 128 MB heap. The most any program of the corpus takes is 234,000 units for one obligation, 145,000 for one subroutine's paths, 783,000 for its proofs and 886,000 for the whole program. `prove-fuzz` makes larger programs than anyone writes: the most in seeds 9137 and 22051 (800 programs each) are 768,000, 35,000,000 (a subroutine its 1.5-second wall-clock budget had already cut short), 5,100,000 and 38,000,000, and the proved obligations of both seeds are the same with the budgets as without. The cross-check is counted the same way: one input of a routine whose ENSURE nests 8 quantifiers visited 5,000,000 positions (7 seconds) before, and the whole cross-check of a routine now takes at most about 2 seconds; the most any routine of the corpus takes is 347,000 (`match-off-by-one`). The fuzzer runs a routine's inputs within its budget in about a second, a program's in about 2 (a program of 2,000 small routines, the first 100 fuzzed: 2 seconds); the most the corpus takes is 400,000 for one routine (kmers' `kmer_counts` in Mojo), 1,300 for one input, and every corpus program's report is the same as before. +The wall-clock budget the Prove tab gives each subroutine (3 seconds, `PROVE_BUDGET_MS` in `src/services/pseudo.ts`) stays behind these as a last resort. It is the caller's: `provePseudo` and `Workspace.proofBudgetMs` take another, and what has to say the same thing on every machine takes none (`pixi run cli prove` unless `--budget-ms` gives one, `library-check`, the tests), so the counted budgets decide alone. The clock is the one part of a proof that depends on the machine: on CI's runner, where the README's example took more than its 3 seconds (it takes 5 milliseconds on the machine this was written on), 4 of its 6 obligations stayed open. A slow or busy device shows the same in the Prove tab: fewer obligations proved, more open, none wrongly proved. Measured on the machine this was written on: 20 nested `ABS` (or a matho pane `f(x) = ||||...x...||||` of 20 bars) took a 1 GB heap down before, and now ends in 0.1 seconds in a 128 MB heap; a subroutine at `UNIT_WORK` takes 0.7 to 2 seconds, one at `PROOF_WORK` about 1 second, and a program at `PROGRAM_WORK` about 4 seconds, each in a 128 MB heap. The most any program of the corpus takes is 234,000 units for one obligation, 145,000 for one subroutine's paths, 783,000 for its proofs and 886,000 for the whole program. `prove-fuzz` makes larger programs than anyone writes: the most in seeds 9137 and 22051 (800 programs each) are 768,000, 35,000,000 (a subroutine its 1.5-second wall-clock budget had already cut short), 5,100,000 and 38,000,000, and the proved obligations of both seeds are the same with the budgets as without. The cross-check is counted the same way: one input of a routine whose ENSURE nests 8 quantifiers visited 5,000,000 positions (7 seconds) before, and the whole cross-check of a routine now takes at most about 2 seconds; the most any routine of the corpus takes is 347,000 (`match-off-by-one`). The fuzzer runs a routine's inputs within its budget in about a second, a program's in about 2 (a program of 2,000 small routines, the first 100 fuzzed: 2 seconds); the most the corpus takes is 400,000 for one routine (kmers' `kmer_counts` in Mojo), 1,300 for one input, and every corpus program's report is the same as before. ## What the compiler does diff --git a/docs/mojo-corpus.md b/docs/mojo-corpus.md index 7666cde..e13bc29 100644 --- a/docs/mojo-corpus.md +++ b/docs/mojo-corpus.md @@ -88,7 +88,7 @@ With `--statements` it writes whole programs instead (20 to 60 lines, 100 per se Two kinds of difference are counted and not reported as failures, because the compiler decides them ([trust.md](trust.md#what-a-translation-says)): numbers that differ only in their last digits (the compiled program's `std.math` is its own; its arithmetic is the pseudo's, since every program is built with contraction off), and, for lines that use `**`, `exp` or `log2`, what those lose in Mojo 1.2.0 (they are good to 9 to 13 digits). What the compiler does wrong on its own is listed in `UPSTREAM` in the fuzzer, with the probe that records it; read that list on every version bump. -Neither task needs or replaces godbolt. A built program that traps is run with its core-dump limit set to 1, so a deliberate crash does not reach the desktop's crash handler. CI runs both in the pinned environment, which brings the compiler: `pixi run ci` runs `mojo-local` and a short `mojo-fuzz`, and the nightly run fuzzes longer ([ci.md](ci.md)). Run both yourself after changing the translator, the checker or the interpreter, and after a version bump. +Neither task needs or replaces godbolt. Both need a C compiler: `mojo build` compiles a program and has the `gcc` or `cc` it finds link it, against a C library no older than glibc 2.34. The pinned environment brings one on Linux (`pixi.toml`), so under `pixi run` the system's is not used; a program that compiles and cannot be linked is reported by `mojo-local` as not linked, the machine's fault and not Mojo's answer, and is not kept in its cache. A built program that traps is run with its core-dump limit set to 1, so a deliberate crash does not reach the desktop's crash handler. CI runs both in the pinned environment, which brings the compiler: `pixi run ci` runs `mojo-local` and a short `mojo-fuzz`, and the nightly run fuzzes longer ([ci.md](ci.md)). Run both yourself after changing the translator, the checker or the interpreter, and after a version bump. ## Reading the status diff --git a/library b/library index 568474d..e5ae090 160000 --- a/library +++ b/library @@ -1 +1 @@ -Subproject commit 568474d52f1dc48227449c2eaee412a11154e272 +Subproject commit e5ae0904cff481b8d183f82e6685f6058b35a043 diff --git a/pixi.lock b/pixi.lock index 02b86e5..0946f68 100644 --- a/pixi.lock +++ b/pixi.lock @@ -19,8 +19,12 @@ environments: packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.46.1-default_hfdba357_102.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.8-hebe6cf0_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/conda-gcc-specs-16.2.0-hf8037ed_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc-16.2.0-hc6a0c74_7.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-16.2.0-h9891192_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.3-py310h44b86e0_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/keyutils-1.6.3-h7cc23a3_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/krb5-1.22.2-hbc21106_2.conda @@ -39,6 +43,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb03c661_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.68.1-h74cf4be_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libpython-3.14.7-hdc7f604_106_cp314.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-16.2.0-h3048135_7.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.22-hebe6cf0_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.53.4-h13e7031_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-16.2.0-h934c35e_7.conda @@ -62,6 +67,9 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/importlib-metadata-9.0.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_client-8.6.3-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/jupyter_core-5.9.1-pyhc90fa1f_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-5.14.0-he073ed8_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/libgcc-devel_linux-64-16.2.0-he3ce08f_107.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/libstdcxx-devel_linux-64-16.2.0-h86e191b_107.conda - conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.1.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/packaging-26.3-pyhc364b38_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pathspec-1.1.1-pyhd8ed1ab_0.conda @@ -70,6 +78,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/python-gil-3.14.7-h4df99d1_106.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python_abi-3.14-9_cp314.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.34-he3f20f0_3.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.4.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/traitlets-5.16.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda @@ -150,6 +159,18 @@ packages: - _openmp_mutex >=4.5 size: 28948 timestamp: 1770939786096 +- conda: https://conda.anaconda.org/conda-forge/linux-64/binutils_impl_linux-64-2.46.1-default_hfdba357_102.conda + sha256: fb7bf36984a37ce7e4714d1d1da0bd0e3bfc679520f5cdc184afc676fd4b5da2 + md5: a0c5e0b7f58c8ceeb08e5bc41251d5a2 + depends: + - ld_impl_linux-64 2.46.1 default_hbd61a6d_102 + - sysroot_linux-64 + - zstd >=1.5.7,<1.6.0a0 + license: GPL-3.0-only + license_family: GPL + run_exports: {} + size: 3713752 + timestamp: 1784214522814 - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda sha256: 1a0d382c515ebf55f8ee1f38c8b81bc95af5c2acc42ad53b66bc5df932032f96 md5: e675fabcf81499adc7edf58124fb1e01 @@ -176,6 +197,44 @@ packages: - c-ares >=1.34.8,<2.0a0 size: 228700 timestamp: 1787169971173 +- conda: https://conda.anaconda.org/conda-forge/linux-64/conda-gcc-specs-16.2.0-hf8037ed_7.conda + sha256: bd4817e82dd8b8c168ff0ed23de4a51af7c60666983f2ee3eb99922806604983 + md5: f81ac04bea74e2911ac8c2d75f8f7c15 + depends: + - gcc_impl_linux-64 >=16.2.0,<16.2.1.0a0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 32833 + timestamp: 1790303294767 +- conda: https://conda.anaconda.org/conda-forge/linux-64/gcc-16.2.0-hc6a0c74_7.conda + sha256: c5f845589fbe63710e4c84cac980faa38dfc969924dc558b7b53a3742a0d53d4 + md5: 00e8f41f290448e696d1012c8a6c1679 + depends: + - conda-gcc-specs + - gcc_impl_linux-64 16.2.0 h9891192_7 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 29649 + timestamp: 1790303398220 +- conda: https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-16.2.0-h9891192_7.conda + sha256: 14acaa661e4c4ef6c65606612b8b08f8f079c6a9e6da3ffcb77056685c675651 + md5: f548b09900fed2701931f8cf925d8b9b + depends: + - binutils_impl_linux-64 >=2.46.1 + - libgcc >=16.2.0 + - libgcc-devel_linux-64 16.2.0 he3ce08f_107 + - libgomp >=16.2.0 + - libsanitizer 16.2.0 h3048135_7 + - libstdcxx >=16.2.0 + - libstdcxx-devel_linux-64 16.2.0 h86e191b_107 + - sysroot_linux-64 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 86085260 + timestamp: 1790303193926 - conda: https://conda.anaconda.org/conda-forge/linux-64/icu-78.3-py310h44b86e0_2.conda sha256: 9f07834f0c546ab14d885ce0366285f61f44e326c0edd1fc63b8294e113ae432 md5: 72a381cbad04f24b1c2a43ef707f45b4 @@ -428,6 +487,20 @@ packages: run_exports: {} size: 10520530 timestamp: 1788383918299 +- conda: https://conda.anaconda.org/conda-forge/linux-64/libsanitizer-16.2.0-h3048135_7.conda + sha256: e383569c331239308a8663facb76208fc0e141131ff3d17c1c2e50f34716a5a9 + md5: 9cdd9af432fa4eb62094aaf6211425bf + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=16.2.0 + - libstdcxx >=16.2.0 + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: + weak: + - libsanitizer 16.2.0 + size: 8115383 + timestamp: 1790303148068 - conda: https://conda.anaconda.org/conda-forge/linux-64/libsodium-1.0.22-hebe6cf0_3.conda sha256: 2f73c05a1ad07e5a2cb43f0b3759cb04505f0c8bef30b43c9db0caa50218bfd8 md5: 2d14868d78e969226e4ba07444c0dd25 @@ -769,6 +842,36 @@ packages: run_exports: {} size: 65503 timestamp: 1760643864586 +- conda: https://conda.anaconda.org/conda-forge/noarch/kernel-headers_linux-64-5.14.0-he073ed8_3.conda + sha256: c4d973dd80ab10028b3e4c58c1c6d8f437bfeaf6069dafb1130dabc550e9caf4 + md5: 1e453742ed8f6b4250c66be1dd6ba4fe + constrains: + - sysroot_linux-64 ==2.34 + license: LGPL-2.0-or-later AND LGPL-2.0-or-later WITH exceptions AND GPL-2.0-or-later + license_family: GPL + run_exports: {} + size: 1410267 + timestamp: 1765578677964 +- conda: https://conda.anaconda.org/conda-forge/noarch/libgcc-devel_linux-64-16.2.0-he3ce08f_107.conda + sha256: f1e7465f6b53f89b325e441c30ad7b19407b051eca556b6d0386ec05513b8e76 + md5: f689d67c3e828fe80ff290d15f361bc5 + depends: + - __unix + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 3099665 + timestamp: 1790302976359 +- conda: https://conda.anaconda.org/conda-forge/noarch/libstdcxx-devel_linux-64-16.2.0-h86e191b_107.conda + sha256: ebbad84bdebd8a9964020d74dcce0f211f1431878c66326f6115dcb59e4c3469 + md5: a713d3c09518a9fc8f603ac0e157263c + depends: + - __unix + license: GPL-3.0-only WITH GCC-exception-3.1 + license_family: GPL + run_exports: {} + size: 23329491 + timestamp: 1790303058123 - conda: https://conda.anaconda.org/conda-forge/noarch/mypy_extensions-1.1.0-pyha770c72_0.conda sha256: 6ed158e4e5dd8f6a10ad9e525631e35cee8557718f83de7a4e3966b1f772c4b1 md5: e9c622e0d00fa24a6292279af3ab6d06 @@ -855,6 +958,22 @@ packages: run_exports: {} size: 18455 timestamp: 1753199211006 +- conda: https://conda.anaconda.org/conda-forge/noarch/sysroot_linux-64-2.34-he3f20f0_3.conda + sha256: d8793b0fab035d128c29a990db0d0e6a96bf6baf4ded4d55a201f614f3d9cd6a + md5: 0e967042d560f3f1af601f08f5f0e31c + depends: + - kernel-headers_linux-64 5.14.0 he073ed8_3 + - tzdata + track_features: + - sysroot_linux-64_2.34 + - sysroot_linux-64_2.34_feature_2 + license: LGPL-2.0-or-later AND LGPL-2.0-or-later WITH exceptions AND GPL-2.0-or-later + license_family: GPL + run_exports: + strong: + - __glibc >=2.34,<3.0.a0 + size: 40779566 + timestamp: 1765578704254 - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.4.1-pyhcf101f3_0.conda sha256: 91cafdb64268e43e0e10d30bd1bef5af392e69f00edd34dfaf909f69ab2da6bd md5: b5325cf06a000c5b14970462ff5e4d58 diff --git a/pixi.toml b/pixi.toml index bd419a5..df583cb 100644 --- a/pixi.toml +++ b/pixi.toml @@ -21,6 +21,14 @@ mojo = "==1.2.0" # the compiler the corpus is pinned to (library/mojo/pin.json) nodejs = ">=26.10.0,<26.11" # the build and tests; the quest processor needs >= 22.6 python = ">=3.14.7,<3.15" # the corpus pipeline and the Dafny pin (standard library only) +# A C compiler, as `cc` and `gcc`: the Mojo compiler has the C compiler it finds link every program, library-check +# builds the C references with it, and one test builds a C harness. CI's image has none (docs/ci.md). Mojo's runtime +# libraries need glibc 2.34, so the C library the compiler links against is pinned to that and not conda-forge's +# default, 2.28, against which no Mojo program links. On macOS the compiler is the system's. +[target.linux-64.dependencies] +gcc = ">=16.2.0,<16.3" +sysroot_linux-64 = ">=2.34,<2.35" + [tasks] # --- build and test -------------------------------------------------------- install = { cmd = "npm install", inputs = ["package.json"], outputs = ["node_modules/.package-lock.json"], description = "Install TypeScript (the only npm dependency)" } diff --git a/scripts/cli.mjs b/scripts/cli.mjs index a452423..abcf1ed 100644 --- a/scripts/cli.mjs +++ b/scripts/cli.mjs @@ -6,10 +6,14 @@ * * pixi run cli check what the language's own reader says * pixi run cli run [--call C] [--input F] run it, print what it prints - * pixi run cli prove [--speco F] every obligation: proved, broken or open + * pixi run cli prove [--speco F] [--budget-ms N] every obligation: proved, broken or open * pixi run cli fuzz [--seed N] [--budget N] the fuzzer's report per routine * pixi run cli translate --to mojo|lingo|matho|speco * + * prove has no time limit here unless --budget-ms gives one (milliseconds for each routine): the prover's counted + * budgets decide alone, so the answer is the same on every machine. The Prove tab of the app gives each routine 3 + * seconds as well, and on a slow machine can leave open what this proves; `--budget-ms 3000` asks for what it does. + * * The language comes from the extension: .pseudo, .lingo, .matho (or .tex), .speco, .mojo, .asm (or .s). Add --json * for machine-readable output. Exit status: 0 when nothing is wrong, 1 when the command found a problem (an error, * a broken obligation, a failing input, a run that stopped), 2 for a usage error. Build first (`pixi run build`): @@ -23,7 +27,7 @@ const ROOT = fileURLToPath(new URL('..', import.meta.url)); const dist = (p) => pathToFileURL(path.join(ROOT, 'dist', p)).href; const usage = (why) => { if (why) console.error(`pplus: ${why}`); - console.error('usage: pplus [--call C] [--input F] [--speco F] [--seed N] [--budget N] [--to LANG] [--json]'); + console.error('usage: pplus [--call C] [--input F] [--speco F] [--seed N] [--budget N] [--budget-ms N] [--to LANG] [--json]'); process.exit(2); }; if (!fs.existsSync(path.join(ROOT, 'dist', 'common', 'workspace.js'))) usage('dist/ is missing; run `pixi run build` first'); @@ -66,6 +70,9 @@ switch (cmd) { } case 'prove': { if (lang !== 'pseudo') usage('prove reads a .pseudo file (with --speco for a separate spec)'); + const clock = opt('budget-ms'); + if (clock !== undefined && !(Number(clock) > 0)) usage('--budget-ms takes a number of milliseconds above 0'); + ws.proofBudgetMs = clock === undefined ? Infinity : Number(clock); const p = ws.proof(); if (!p) { out({ error: 'the pseudo does not parse' }, ['pplus: the pseudo does not parse; run `pplus check` first']); process.exit(1); } const { report } = p; diff --git a/scripts/dafny.py b/scripts/dafny.py index b29afa9..ef01ac8 100644 --- a/scripts/dafny.py +++ b/scripts/dafny.py @@ -13,6 +13,12 @@ it into .tools/dafny-/ (ignored by git). The zip is self-contained: it builds Dafny was tested with, so the solver is pinned too. To move to another Dafny, change VERSION and every hash in RELEASES in one commit, and expect every stored proof answer to be asked again. +One thing the zip does not carry is ICU, which .NET loads for culture-aware text; on a machine without it (a slim +container image) Dafny aborts before it prints its version. Dafny is run here with DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 +(`env()` below), which tells .NET not to look for ICU: verifying a program compares no text by culture, so the +verdicts are the same and the machine needs nothing installed. Whatever else runs the executable (scripts/library.mjs, +tests/dafny.test.mjs) sets the same variable. + Standard library only, like the corpus pipeline. """ from __future__ import annotations @@ -98,16 +104,24 @@ def fetch(quiet: bool = False) -> Path: return EXE +def env() -> dict[str, str]: + """The environment Dafny runs in: this one, and .NET told not to look for ICU (the module's docstring).""" + return {**os.environ, "DOTNET_SYSTEM_GLOBALIZATION_INVARIANT": "1"} + + def main(argv: list[str]) -> int: if argv[:1] == ["fetch"]: fetch() - print(subprocess.run([str(EXE), "--version"], capture_output=True, text=True).stdout.strip()) + v = subprocess.run([str(EXE), "--version"], capture_output=True, text=True, env=env()) + if v.returncode != 0 or not v.stdout.startswith(VERSION): + raise SystemExit(f"dafny: the fetched Dafny does not start here (exit {v.returncode}): {(v.stderr or v.stdout).strip()[:400]}") + print(v.stdout.strip()) return 0 if argv[:1] == ["path"]: print(fetch(quiet=True)) return 0 exe = fetch() - return subprocess.run([str(exe), *argv]).returncode + return subprocess.run([str(exe), *argv], env=env()).returncode if __name__ == "__main__": diff --git a/scripts/library.mjs b/scripts/library.mjs index d5d6017..028cb3e 100644 --- a/scripts/library.mjs +++ b/scripts/library.mjs @@ -473,7 +473,8 @@ export async function checkLibrary(dir = LIB, only = null) { } rec.checks.pseudo = caseFail ? bad(caseFail) : ok(`${calls.length} case(s) run`); // prover - const pr = svc.provePseudo(st); + // no time on the clock: what is recorded is the counted budgets' answer, the same on every machine + const pr = svc.provePseudo(st, { budgetMs: Infinity }); const obs = pr.units.flatMap((u) => u.obligations); const broken = obs.filter((o) => o.status === 'broken'); const notProved = obs.filter((o) => o.status !== 'proved'); @@ -591,7 +592,8 @@ export async function checkLibrary(dir = LIB, only = null) { const src = r.proof.source; const flags = ['verify', '--resource-limit', '20000000']; if (dafny) { - const p = spawnSync(dafny, [...flags, path.join(rdir, r.proof.file)], { encoding: 'utf8', timeout: 600000 }); + // without ICU on the machine .NET aborts unless told not to look for it (scripts/dafny.py) + const p = spawnSync(dafny, [...flags, path.join(rdir, r.proof.file)], { encoding: 'utf8', timeout: 600000, env: { ...process.env, DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' } }); const summary = /Dafny program verifier finished with (.*)/.exec(p.stdout || '')?.[1] || (p.stdout || p.stderr || '').trim().split('\n').pop(); const pass = p.status === 0 && / 0 errors/.test(summary); rec.proof = pass ? ok(`Dafny ${dafnyVersion}: ${summary}`) : bad(`Dafny ${dafnyVersion}: ${summary}`); diff --git a/scripts/mojo/local.mjs b/scripts/mojo/local.mjs index 2592439..0118987 100644 --- a/scripts/mojo/local.mjs +++ b/scripts/mojo/local.mjs @@ -176,6 +176,14 @@ const exec = (cmd, args, { cwd, input = '', timeout = 60000 } = {}) => new Promi const SIGNALS = os.constants.signals; const HAVE_PRLIMIT = spawnSync('prlimit', ['--version'], { encoding: 'utf8' }).status === 0; +/** + * `mojo build` compiles a program and then has the C compiler it finds on the PATH link it. When that step fails (no + * C compiler, or one that links against a C library older than Mojo's runtime needs) the compiler never said + * anything about the program: the answer is the machine's, so it is not kept in the cache, not held against the + * checker or godbolt, and reported as what it is. + */ +const NOT_LINKED = /unable to find suitable c compiler|failed to link executable/; + /** Build one program and run it: { compiled, buildStderr, executed, exitCode, stdout, stderr, timedOut }, in godbolt's shape. */ async function buildAndRun(p, workdir) { const dir = fs.mkdtempSync(path.join(workdir, 'p-')); @@ -186,7 +194,8 @@ async function buildAndRun(p, workdir) { const extra = p.args.split(/\s+/).filter(Boolean); const b = await exec('mojo', ['build', ...extra, 'example.mojo', '-o', 'example'], { cwd: dir, timeout: 180000 }); if (b.status !== 0 || !fs.existsSync(path.join(dir, 'example'))) { - return { compiled: false, buildStderr: app(b.stderr || b.stdout), executed: false, exitCode: null, stdout: '', stderr: '', timedOut: b.timedOut }; + const buildStderr = app(b.stderr || b.stdout); + return { compiled: false, buildStderr, executed: false, exitCode: null, stdout: '', stderr: '', timedOut: b.timedOut, ...(NOT_LINKED.test(buildStderr) ? { notLinked: true } : {}) }; } // a trap must not become a core dump on someone's desktop: limit 1 tells the kernel not to pipe it const r = HAVE_PRLIMIT ? await exec('prlimit', ['--core=1:1', './example'], { cwd: dir, input: p.stdin, timeout: 30000 }) @@ -300,7 +309,7 @@ const jobs = Math.max(1, Number(opt('jobs', '2')) || 2); const useCache = !flag('no-cache'); const findings = []; -const counts = { programs: programs.length, compiled: 0, rejected: 0, cached: 0, interpreterRan: 0, interpreterDeclined: 0, godboltCompared: 0 }; +const counts = { programs: programs.length, compiled: 0, rejected: 0, notLinked: 0, cached: 0, interpreterRan: 0, interpreterDeclined: 0, godboltCompared: 0 }; let next = 0; async function worker() { while (next < programs.length) { @@ -309,7 +318,8 @@ async function worker() { const file = path.join(cacheDir, `${key}.json`); let r = useCache && fs.existsSync(file) ? readJson(file) : null; if (r) { counts.cached++; for (const k of ['stdout', 'stderr', 'buildStderr']) r[k] = String(r[k] ?? '').replace(/(?:\/[^\s:"']+)?\/example\.mojo/g, '/app/example.mojo'); } - else { r = await buildAndRun(p, workdir); if (useCache && !r.timedOut) fs.writeFileSync(file, `${JSON.stringify(r)}\n`); } + else { r = await buildAndRun(p, workdir); if (useCache && !r.timedOut && !r.notLinked) fs.writeFileSync(file, `${JSON.stringify(r)}\n`); } + if (r.notLinked) { counts.notLinked++; findings.push({ question: 'linker', id: p.id, detail: `not linked, so the compiler's answer is not known: ${firstError(r.buildStderr)}` }); continue; } if (r.compiled) counts.compiled++; else counts.rejected++; findings.push(...judge(p, r)); if (p.interp === 'ran') counts.interpreterRan++; @@ -331,7 +341,8 @@ if (flag('json')) console.log(JSON.stringify({ compiler: `Mojo ${version}`, coun else { console.log(`Mojo ${version} on this machine: ${counts.programs} programs (${counts.compiled} compiled, ${counts.rejected} rejected, ${counts.cached} from the cache)`); console.log(`the interpreter ran ${counts.interpreterRan} of them and declined ${counts.interpreterDeclined}; ${counts.godboltCompared} compared with godbolt's stored answer`); - for (const q of ['checker', 'interpreter', 'translator', 'godbolt']) { + if (counts.notLinked) console.log(`\n${counts.notLinked} program(s) were compiled and could not be linked. That is this machine's C compiler, which mojo build calls to link, not Mojo's answer: under pixi it is the pinned gcc and C library (pixi.toml, docs/ci.md).`); + for (const q of ['linker', 'checker', 'interpreter', 'translator', 'godbolt']) { const fs_ = unexpected.filter((f) => f.question === q); if (!fs_.length) continue; console.log(`\n${q}: ${fs_.length}`); diff --git a/src/common/workspace.ts b/src/common/workspace.ts index 4c6ed1c..2eae5fb 100644 --- a/src/common/workspace.ts +++ b/src/common/workspace.ts @@ -247,12 +247,18 @@ export class Workspace { /** The prover's own failure, said on the speco pane (proof() is then null, as for a program that does not check). */ proofFault: Diagnostic | null = null; + /** + * The time on the clock a proof may take for each routine, in milliseconds; left unset, the Prove tab's + * (PROVE_BUDGET_MS in services/pseudo.ts). The command line sets Infinity, so its answer is the counted budgets' + * alone and the same on every machine. + */ + proofBudgetMs: number | undefined = undefined; proof(): ProofResult | null { const p = this.parsedPseudo(); if (!p || !p.ok) return null; const m = this.parsedSpeco(); - const key = JSON.stringify([this.source('pseudo'), this.source('speco')]); + const key = JSON.stringify([this.source('pseudo'), this.source('speco'), String(this.proofBudgetMs)]); const r = this.cached('proof', key, () => { try { return { result: this.proofOf(p, m), fault: null }; } catch (e) { return { result: null, fault: faultDiagnostic('speco', e) }; } }); @@ -262,10 +268,10 @@ export class Workspace { private proofOf(p: Parsed, m: Parsed | null): ProofResult { if (m && m.ok && m.program!.spec.units.length) { - const r = proveAgainst(m, this.source('pseudo')); + const r = proveAgainst(m, this.source('pseudo'), { budgetMs: this.proofBudgetMs }); return { report: r.report, wovenSource: r.woven.source, wovenDiagnostics: [...r.woven.diagnostics, ...r.study.diagnostics.filter((d) => d.severity === 'error').map((d) => ({ ...d, lang: 'speco' as LangId, message: `In the woven pseudocode, line ${d.line}: ${d.message}` }))], study: r.study }; } - return { report: provePseudo(p), wovenSource: null, wovenDiagnostics: [], study: p }; + return { report: provePseudo(p, { budgetMs: this.proofBudgetMs }), wovenSource: null, wovenDiagnostics: [], study: p }; } /** diff --git a/src/services/pseudo.ts b/src/services/pseudo.ts index 6a4db8e..4c9a485 100644 --- a/src/services/pseudo.ts +++ b/src/services/pseudo.ts @@ -112,10 +112,26 @@ export function runPseudo(parsed: Parsed, options: RunOptions): Trace): ProofReport { +/** + * The time on the clock the Prove tab gives each routine, twice: to the prover, then to the generated inputs that + * cross-check it. It is the last resort behind the counted budgets (docs/limits.md), there so a page never waits long + * on one routine, and it makes the answer depend on the machine: where the work takes longer than this, obligations + * the prover would have proved stay open, and inputs that would have broken one are not tried. Nothing is ever + * called proved because of it. + */ +export const PROVE_BUDGET_MS = 3000; +/** + * `budgetMs`: the time on the clock for each routine (PROVE_BUDGET_MS when left out). A tool whose answer has to be + * the same on every machine (the command line, library-check, a test) passes Infinity: the counted budgets then + * decide alone, and they run out at the same point everywhere. + */ +export interface ProveOptions { budgetMs?: number } + +export function provePseudo(parsed: Parsed, options: ProveOptions = {}): ProofReport { const s = parsed.program; if (!s || !parsed.ok) return { units: [], counts: { proved: 0, broken: 0, open: 0, total: 0 } }; - const r = verify(s.program, { budgetMs: 3000 }); + const budgetMs = options.budgetMs ?? PROVE_BUDGET_MS; + const r = verify(s.program, { budgetMs }); // Cross-check unproved obligations against generated inputs: an obligation // a run breaks is "broken", the rest stay "open". const brokenKeys = new Set(); @@ -127,7 +143,7 @@ export function provePseudo(parsed: Parsed): ProofReport { if (left <= 0) break; try { // and the same wall-clock budget the verifier has, per routine: a slow routine is tried on fewer inputs - const pr = checkProperties(s.program, sub.name, { count: 150, budgetMs: 3000, maxWork: Math.min(left, CROSS_CHECK_ROUTINE) }); + const pr = checkProperties(s.program, sub.name, { count: 150, budgetMs, maxWork: Math.min(left, CROSS_CHECK_ROUTINE) }); left -= (pr as { work?: number }).work ?? 0; // a failure that says which half of a contract it broke (an INVARIANT before the first pass or after one, a // DECREASES below 0 or not smaller) counts against that obligation only diff --git a/src/services/speco.ts b/src/services/speco.ts index 0819b33..11d84d8 100644 --- a/src/services/speco.ts +++ b/src/services/speco.ts @@ -21,7 +21,7 @@ * speco document, so existing examples show up in the pane. */ import type { Diagnostic, LanguageService, Lowered, Parsed, ProofReport } from '../common/types.js'; -import { studyOf, provePseudo } from './pseudo.js'; +import { studyOf, provePseudo, type ProveOptions } from './pseudo.js'; import { SPECO_SYMBOLS } from '../common/reference.js'; export interface SpecLoop { selector: string; /** 1-based line of the LOOP header in the speco source */ line?: number; lines: Array<{ kind: 'INVARIANT' | 'DECREASES'; text: string; line: number }> } @@ -398,10 +398,10 @@ export function parseSpeco(source: string): Parsed { } /** Prove a pseudocode program against this spec. Obligation lines are reported in speco's numbering where a claim came from speco. */ -export function proveAgainst(parsed: Parsed, pseudo: string): { report: ProofReport; woven: Woven; study: Parsed } { +export function proveAgainst(parsed: Parsed, pseudo: string, options: ProveOptions = {}): { report: ProofReport; woven: Woven; study: Parsed } { const woven = weave(pseudo, parsed.program!.spec); const st = studyOf(woven.source); - const report = provePseudo(st); + const report = provePseudo(st, options); const back = new Map(woven.map.map((m) => [m.to, m.from])); for (const u of report.units) for (const o of u.obligations) { const from = back.get(o.line); diff --git a/tests/dafny.test.mjs b/tests/dafny.test.mjs index ddf5d83..62f101d 100644 --- a/tests/dafny.test.mjs +++ b/tests/dafny.test.mjs @@ -23,11 +23,13 @@ test('the Dafny pin\'s own tests pass (python3)', { skip: py.status !== 0 && !pr const exe = ROOT + '.tools/dafny-4.11.0/dafny/dafny'; test('the fetched Dafny is the pinned one, and verifies a program', { skip: fs.existsSync(exe) || process.env.CI ? false : 'Dafny has not been fetched (pixi run dafny-fetch)' }, () => { assert.ok(fs.existsSync(exe), 'Dafny has not been fetched, and CI must run this: pixi run dafny-fetch'); - const v = spawnSync(exe, ['--version'], { encoding: 'utf8' }); - assert.match(v.stdout, /^4\.11\.0\+/); + // as scripts/dafny.py runs it: .NET told not to look for ICU, which a slim image does not have + const env = { ...process.env, DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: '1' }; + const v = spawnSync(exe, ['--version'], { encoding: 'utf8', env }); + assert.match(v.stdout, /^4\.11\.0\+/, v.stderr); const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pplus-dafny-')); fs.writeFileSync(dir + '/max.dfy', 'method Max(a: int, b: int) returns (m: int) ensures m >= a && m >= b { if a > b { m := a; } else { m := b; } }\n'); - const r = spawnSync(exe, ['verify', dir + '/max.dfy'], { encoding: 'utf8' }); + const r = spawnSync(exe, ['verify', dir + '/max.dfy'], { encoding: 'utf8', env }); fs.rmSync(dir, { recursive: true, force: true }); assert.equal(r.status, 0, r.stdout + r.stderr); assert.match(r.stdout, /1 verified, 0 errors/); diff --git a/tests/engine-solver.test.mjs b/tests/engine-solver.test.mjs index 4b91689..a62d8b3 100644 --- a/tests/engine-solver.test.mjs +++ b/tests/engine-solver.test.mjs @@ -11,6 +11,7 @@ import { } from '../dist/engines/pseudo/logic.js'; import { verify } from '../dist/engines/pseudo/vcgen.js'; import { studyOf } from '../dist/services/pseudo.js'; +import { NO_CLOCK } from './helpers/pace.mjs'; const x = sym('X', 'int'); const y = sym('Y', 'int'); const z = sym('Z', 'int'); const i = sym('I', 'int'); const j = sym('J', 'int'); @@ -141,7 +142,7 @@ test('the solver never proves a formula that a small assignment breaks (random l const statuses = (src) => { const st = studyOf(src); assert.ok(st.ok, 'the program parses'); - return verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations.map((o) => `${o.kind}@${o.line}:${o.status}`)); + return verify(st.program.program, NO_CLOCK).units.flatMap((u) => u.obligations.map((o) => `${o.kind}@${o.line}:${o.status}`)); }; test('vcgen: a counting loop with its invariant proves its ENSURE and every index', () => { @@ -195,7 +196,7 @@ test('vcgen: FILL and RANDINT ask whether their arguments are allowed', () => { test('vcgen: integer arithmetic carries the note on what "proved" assumes about whole numbers', () => { const st = studyOf(['FUNCTION INC(N : INTEGER) RETURNS INTEGER', ' ENSURE RESULT > N', ' RETURN N + 1', 'ENDFUNCTION'].join('\n')); - const u = verify(st.program.program, {}).units.find((x) => x.name === 'INC'); + const u = verify(st.program.program, NO_CLOCK).units.find((x) => x.name === 'INC'); assert.ok(u.obligations.every((o) => o.status === 'proved')); assert.ok(u.notes.some((nt) => nt.kind === 'int-model'), 'the int-model note is missing'); }); diff --git a/tests/exercises.test.mjs b/tests/exercises.test.mjs index 792b7e9..9bf7fd9 100644 --- a/tests/exercises.test.mjs +++ b/tests/exercises.test.mjs @@ -34,6 +34,7 @@ test('the template itself is a valid exercise', () => { const tr = s.trace(); assert.ok(tr && !tr.error, s.runError || tr?.error?.message); assert.ok(tr.output.at(-1).endsWith(`returned ${template.expect}`)); + s.proofBudgetMs = Infinity; // the counted budgets' answer, the same on every machine assert.equal(s.proof().report.counts.open + s.proof().report.counts.broken, 0); }); diff --git a/tests/helpers/pace.mjs b/tests/helpers/pace.mjs new file mode 100644 index 0000000..dfe6bba --- /dev/null +++ b/tests/helpers/pace.mjs @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: 2026 Scientific Computing Studio, LLC +// SPDX-License-Identifier: Apache-2.0 +// Time limits that mean the same thing on every machine. A test that says "this takes under 2 seconds" is there to +// catch work that grows with the square of its input (seconds become minutes), and its limit was written on one +// machine. On CI's runner, a small virtual machine that runs eight test files at once, the same correct code took 4 +// to 12 times the limits, and six tests failed with nothing wrong. So a limit here is: +// +// - measured in the processor time this process used (process.cpuUsage), not time on the clock, so the other test +// files that share the processors do not count against it; +// - multiplied by this machine's pace: how many times longer a fixed piece of work (`work` below) takes here than +// the REFERENCE_MS it took where the limits were written. The pace is taken before and after what is timed, and +// the larger counts. It is never below 1, so no limit gets tighter on a faster machine, and on the machine the +// limits were written on they are what they were. +// +// A failure says the time, the limit and the pace. PPLUS_PACE_LOG=1 prints every measurement that passed too. +import assert from 'node:assert/strict'; + +/** Milliseconds `work()` takes, at its best, on the machine the tests' limits were written on. */ +export const REFERENCE_MS = 2.3; + +/** A little of what the engines do: cut names out of text, make small objects, count in a Map, sort. */ +function work() { + let sum = 0; + const names = new Map(); + for (let i = 0; i < 40000; i++) { + const word = `v${i % 997}`; + const node = { kind: 'name', text: word, line: i, at: [i, i + word.length] }; + names.set(word, (names.get(word) ?? 0) + node.at[1] - node.at[0]); + sum += word.charCodeAt(word.length - 1); + } + const list = [...names].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1)); + return sum + list.length; +} + +/** Processor time this process has used, in milliseconds (every thread of it: the collector and the compiler too). */ +const cpuMs = () => { const u = process.cpuUsage(); return (u.user + u.system) / 1000; }; + +let warm = false; +/** How many times longer the fixed work takes here, now, than on the reference machine; at least 1. */ +export function pace() { + if (!warm) { for (let i = 0; i < 20; i++) work(); warm = true; } + let best = Infinity; + for (let i = 0; i < 5; i++) { const t0 = cpuMs(); work(); best = Math.min(best, cpuMs() - t0); } + return Math.max(1, best / REFERENCE_MS); +} + +/** The processor time `fn` takes, in milliseconds, and the pace around it. */ +export function measure(fn) { + const before = pace(); + const t0 = cpuMs(); + fn(); + const ms = cpuMs() - t0; + return { ms, pace: Math.max(before, pace()) }; +} + +/** Whether a measurement is inside a limit written for the reference machine, as a message when it is not. */ +export function over(m, limitMs, what) { + const line = `${what}: ${m.ms.toFixed(1)} ms of processor time; the limit is ${limitMs} ms${m.pace > 1 ? ` x ${m.pace.toFixed(1)}, this machine's pace (tests/helpers/pace.mjs)` : ''}`; + if (process.env.PPLUS_PACE_LOG) console.log(`pace: ${line}`); + return m.ms < limitMs * m.pace ? null : line; +} + +/** Run `fn` and fail unless it took under `limitMs` of processor time at this machine's pace. */ +export function within(limitMs, what, fn) { + const failed = over(measure(fn), limitMs, what); + assert.ok(!failed, failed ?? ''); +} + +/** + * For the prover in a test: no time on the clock for a routine. Its counted budgets then decide alone (docs/limits.md), + * so what a test finds proved, open or broken is the same on every machine. With the Prove tab's 3 seconds, a slow + * runner left open what the tests expect proved. + */ +export const NO_CLOCK = Object.freeze({ budgetMs: Infinity }); diff --git a/tests/library.test.mjs b/tests/library.test.mjs index b75d3b6..554f185 100644 --- a/tests/library.test.mjs +++ b/tests/library.test.mjs @@ -12,6 +12,7 @@ import { studyOf, runPseudo, provePseudo } from '../dist/services/pseudo.js'; import { pseudoToMojo } from '../dist/sync/pseudo-to-mojo.js'; import { checkMojo } from '../dist/mojo/check.js'; import { runMojo } from '../dist/mojo/interp.js'; +import { NO_CLOCK } from './helpers/pace.mjs'; test('the registry is generated from the library checkout and is up to date', () => { assert.ok(fs.existsSync('library/modules'), 'library/ is not checked out: git submodule update --init'); @@ -82,7 +83,7 @@ test('a run steps over a library routine, and a broken REQUIRE is reported at th }); test('the prover uses a routine\'s REQUIRE at the call and its ENSURE afterwards, without proving it again', () => { - const r = provePseudo(studyOf('USE seq.HammingDistance\nFUNCTION F(S : STRING) RETURNS INTEGER\n ENSURE RESULT <= LENGTH(S)\n RETURN HammingDistance(S, S)\nENDFUNCTION\n')); + const r = provePseudo(studyOf('USE seq.HammingDistance\nFUNCTION F(S : STRING) RETURNS INTEGER\n ENSURE RESULT <= LENGTH(S)\n RETURN HammingDistance(S, S)\nENDFUNCTION\n'), NO_CLOCK); assert.deepEqual(r.units.map((u) => u.name), ['F'], 'library routines are not units of the program'); const obs = r.units[0].obligations; assert.equal(obs.find((o) => o.kind === 'require').status, 'proved'); diff --git a/tests/numeric.test.mjs b/tests/numeric.test.mjs index 6482628..5abc627 100644 --- a/tests/numeric.test.mjs +++ b/tests/numeric.test.mjs @@ -8,6 +8,7 @@ import assert from 'node:assert/strict'; import { studyOf, runPseudo, provePseudo } from '../dist/services/pseudo.js'; import { fma, exactDecimal, nextAfter, ulp, roundDyadic, wrap } from '../dist/engines/pseudo/numeric.js'; import { intRange, SIZED, formatFloat32 } from '../dist/engines/pseudo/lang.js'; +import { NO_CLOCK } from './helpers/pace.mjs'; const run = (src) => { const st = studyOf(src); @@ -144,7 +145,7 @@ test('names: a program may still call its variable E or its function GCD, and it }); test('the prover: sized arithmetic carries an overflow obligation; FLOAT32 is a function of its own; USE IEEE proves nothing', () => { - const report = (src) => provePseudo(studyOf(src)).units.flatMap((u) => u.obligations.map((o) => ({ ...o, unit: u.name, notes: u.notes }))); + const report = (src) => provePseudo(studyOf(src), NO_CLOCK).units.flatMap((u) => u.obligations.map((o) => ({ ...o, unit: u.name, notes: u.notes }))); const guarded = report('FUNCTION Inc(X : INT8) RETURNS INT8\n REQUIRE X < 127\n RETURN X + 1\nENDFUNCTION\n'); assert.deepEqual(guarded.filter((o) => o.kind === 'overflow').map((o) => o.status), ['proved']); const open = report('FUNCTION Inc(X : INT8) RETURNS INT8\n RETURN X + 1\nENDFUNCTION\n'); @@ -260,7 +261,7 @@ test('sized results that were not checked: ABS, GCD, BIT_NOT of a UINT64, ADD, a assert.equal(r.error?.kind, 'overflow'); assert.deepEqual(out('DECLARE I : INT8\nFOR I <- 125 TO 128\n OUTPUT I\nNEXT I'), ['125', '126', '127']); // the prover asks the same obligation where the run now stops - const rep = provePseudo(studyOf('FUNCTION F(X : INT8) RETURNS INT8\n RETURN ABS(X)\nENDFUNCTION')); + const rep = provePseudo(studyOf('FUNCTION F(X : INT8) RETURNS INT8\n RETURN ABS(X)\nENDFUNCTION'), NO_CLOCK); assert.ok(rep.units.flatMap((u) => u.obligations).some((o) => o.kind === 'overflow' && /ABS\(X\) fits INT8/.test(o.label) && o.status !== 'proved')); }); diff --git a/tests/prover-limits.test.mjs b/tests/prover-limits.test.mjs index b7e44cd..a41bb64 100644 --- a/tests/prover-limits.test.mjs +++ b/tests/prover-limits.test.mjs @@ -5,12 +5,14 @@ // never "proved" and never a missing obligation (docs/limits.md). import test from 'node:test'; import assert from 'node:assert/strict'; +import fs from 'node:fs'; import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { studyOf, provePseudo } from '../dist/services/pseudo.js'; import { verify } from '../dist/engines/pseudo/vcgen.js'; import { prove } from '../dist/engines/pseudo/solver.js'; import { sym, num, cmp, ite, neg, add, key, show, TOO_LARGE, MAX_KEY } from '../dist/engines/pseudo/logic.js'; +import { NO_CLOCK, within } from './helpers/pace.mjs'; const DIST = fileURLToPath(new URL('../dist/', import.meta.url)); const absOf = (n) => `${'ABS('.repeat(n)}X${')'.repeat(n)}`; @@ -30,8 +32,9 @@ test('nested ABS and a matho pane of 20 bars are proved in a 128 MB heap: open, const { studyOf, provePseudo } = await import(process.env.DIST + 'services/pseudo.js'); const { Workspace } = await import(process.env.DIST + 'common/workspace.js'); const src = 'FUNCTION F(X : INTEGER) RETURNS INTEGER\\n ENSURE RESULT >= 0\\n RETURN ${absOf(20)}\\nENDFUNCTION\\n'; - const r = provePseudo(studyOf(src)); + const r = provePseudo(studyOf(src), { budgetMs: Infinity }); const ws = new Workspace({ sources: { matho: 'f(x) = ${'|'.repeat(20)}x${'|'.repeat(20)}' }, lang: 'matho' }); + ws.proofBudgetMs = Infinity; const m = ws.proof(); console.log(JSON.stringify({ counts: r.counts, ob: r.units[0].obligations.map((o) => [o.kind, o.status, o.reason]), matho: m && m.report.counts, fault: ws.proofFault })); `); @@ -75,24 +78,48 @@ test('prove: past its work the answer is unknown, too large to prove here, and n test('a subroutine past the checker\'s work is partial and says so: nothing in it proved, the same answer every time', () => { const p = program(nestedFor(30)); - const r1 = verify(p, { budgetMs: 600000 }); - const r2 = verify(p, { budgetMs: 600000 }); + const r1 = verify(p, NO_CLOCK); + const r2 = verify(p, NO_CLOCK); assert.equal(r1.units[0].partial, true); const note = r1.units[0].notes.find((n) => n.kind === 'too-large'); assert.match(note.text, /too large to prove here.*nothing in it is marked proved/); assert.deepEqual(r2.units[0].obligations.map((o) => [o.kind, o.line, o.status]), r1.units[0].obligations.map((o) => [o.kind, o.line, o.status])); - const report = provePseudo(studyOf(nestedFor(30))); + const report = provePseudo(studyOf(nestedFor(30)), NO_CLOCK); assert.equal(report.counts.proved, 0); // two levels of the same loops are proved as before - const small = verify(program(nestedFor(2))); + const small = verify(program(nestedFor(2)), NO_CLOCK); assert.equal(small.units[0].partial, false); assert.ok(small.counts.proved >= 1); }); +test('the time on the clock is the caller\'s: run out, it leaves a subroutine partial and proves less, never more; the command line has none', async () => { + const { Workspace } = await import('../dist/common/workspace.js'); + const src = fs.readFileSync(new URL('../library/pseudo/programs/19-max-subarray.pseudo', import.meta.url), 'utf8'); + const whole = provePseudo(studyOf(src), NO_CLOCK); + assert.deepEqual(whole.counts, { proved: 6, broken: 0, open: 0, total: 6 }); + // a clock that has already run out (what 3 seconds were on CI's runner): the subroutine with the loop is partial + const cut = provePseudo(studyOf(src), { budgetMs: -1 }); + assert.equal(cut.units.find((u) => u.name === 'MAX_SUBARRAY').partial, true); + assert.ok(cut.counts.proved < 6 && cut.counts.broken === 0, JSON.stringify(cut.counts)); + // a Workspace hands its own on, and proves again when it changes + const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' }); + ws.proofBudgetMs = -1; + assert.ok(ws.proof().report.counts.proved < 6); + ws.proofBudgetMs = Infinity; + assert.equal(ws.proof().report.counts.proved, 6); + // the command line has no clock unless it is given one, and refuses one that is not a time + const cli = (...args) => spawnSync(process.execPath, ['scripts/cli.mjs', 'prove', 'library/pseudo/programs/19-max-subarray.pseudo', ...args], { cwd: fileURLToPath(new URL('..', import.meta.url)), encoding: 'utf8' }); + assert.equal(cli().stdout.trim().split('\n').pop(), '6 proved, 0 broken, 0 open'); + assert.equal(cli('--budget-ms', '600000').stdout.trim().split('\n').pop(), '6 proved, 0 broken, 0 open'); + const bad = cli('--budget-ms', 'soon'); + assert.equal(bad.status, 2); + assert.match(bad.stderr, /--budget-ms takes a number of milliseconds above 0/); +}); + test('past a subroutine\'s proving work, every obligation is still listed, open and too large to prove here', () => { const n = 40; const src = fn('X : INTEGER', `${Array.from({ length: n }, (_, k) => ` ASSERT ${absOf(12).replace('X)', `X - ${k})`)} >= 0`).join('\n')}\n RETURN X`); - const r = verify(program(src), { budgetMs: 600000 }); + const r = verify(program(src), NO_CLOCK); const obs = r.units[0].obligations; assert.equal(obs.length, n); assert.ok(obs.every((o) => o.status === 'unknown')); @@ -101,7 +128,7 @@ test('past a subroutine\'s proving work, every obligation is still listed, open test('past a program\'s work, the subroutines after it are partial and say so', () => { const src = [nestedFor(30, 'A'), nestedFor(30, 'B'), nestedFor(30, 'C'), fn('X : INTEGER', ' RETURN X', ' ENSURE RESULT == X\n').replace('FUNCTION F(', 'FUNCTION D(')].join('\n'); - const r = verify(program(src), { budgetMs: 600000 }); + const r = verify(program(src), NO_CLOCK); assert.deepEqual(r.units.map((u) => u.partial), [true, true, true, true]); for (const u of r.units) assert.ok(u.notes.some((n) => n.kind === 'too-large'), u.name); assert.equal(r.counts.proved, 0); @@ -146,12 +173,13 @@ test('checkProperties: maxWork bounds the steps and quantifier positions of all test('the Prove tab\'s cross-check of a routine whose ENSURE nests quantifiers ends within its budget of work', async () => { const { CROSS_CHECK_ROUTINE } = await import('../dist/services/pseudo.js'); - const t0 = Date.now(); - const r = provePseudo(studyOf(quantified(8))); - assert.equal(r.counts.broken, 0); + // before the budget one input of this routine visited 5,000,000 positions (about 7 seconds); the limit is + // processor time at this machine's pace (tests/helpers/pace.mjs), with no time on the prover's clock + within(30000, 'the cross-check of 8 nested quantifiers', () => { + const r = provePseudo(studyOf(quantified(8)), NO_CLOCK); + assert.equal(r.counts.broken, 0); + }); assert.ok(CROSS_CHECK_ROUTINE <= 1_000_000); - // before the budget one input of this routine visited 5,000,000 positions (about 7 seconds) - assert.ok(Date.now() - t0 < 30000); }); /* ---------- the fuzzer (Workspace.fuzz, the Tests view) ---------- */ @@ -235,13 +263,14 @@ test('a value built from thousands of operations one inside another becomes an u const { Workspace } = await import('../dist/common/workspace.js'); // 60 lines of 390-term sums: a value 23,000 operations deep, which ran the prover past the engine's stack const src = fn('X : INTEGER', ` DECLARE T : INTEGER\n T <- 0\n${` T <- T${' + X'.repeat(390)}\n`.repeat(60)} ASSERT X == X\n RETURN T`, ' REQUIRE X >= 0\n ENSURE RESULT >= 0\n'); - const r = verify(program(src), { budgetMs: 600000 }); + const r = verify(program(src), NO_CLOCK); assert.equal(r.units[0].partial, false); assert.match(r.units[0].notes.find((n) => n.kind === 'too-deep').text, new RegExp(`more than ${MAX_DEPTH} operations one inside another`)); // what does not depend on the value is still proved, and what does is open assert.deepEqual(r.units[0].obligations.map((o) => [o.kind, o.status]), [['assert', 'proved'], ['ensure', 'unknown']]); const ws = new Workspace({ sources: { pseudo: src }, lang: 'pseudo' }); assert.ok(!ws.allDiagnostics().some((d) => d.id === 'engine-stop')); + ws.proofBudgetMs = Infinity; assert.equal(ws.proof().report.counts.total, 2); }); diff --git a/tests/prover-soundness.test.mjs b/tests/prover-soundness.test.mjs index ebdf998..33db45e 100644 --- a/tests/prover-soundness.test.mjs +++ b/tests/prover-soundness.test.mjs @@ -26,6 +26,7 @@ import { verify } from '../dist/engines/pseudo/vcgen.js'; import { checkProperties } from '../dist/engines/pseudo/testing.js'; import { PROBLEMS } from '../dist/common/problems.js'; import { LESSONS, solutionOf } from '../dist/curriculum/index.js'; +import { NO_CLOCK } from './helpers/pace.mjs'; const PROVER_DIR = 'library/pseudo/prover'; const CORPUS_DIR = 'library/pseudo/programs'; @@ -65,7 +66,7 @@ function unsoundIn(src) { const st = studyOf(src); if (!st.ok) return { parsed: false, bad: [], proved: [] }; const program = st.program.program; - const r = verify(program, { budgetMs: 3000 }); + const r = verify(program, NO_CLOCK); const proved = r.units.filter((u) => !u.partial).flatMap((u) => u.obligations.filter((o) => o.status === 'proved')); const failures = []; for (const sub of program.subs.values()) { @@ -98,7 +99,7 @@ test('regression programs: an obligation an unsound rule once proved is never pr } const st = studyOf(src); assert.ok(st.ok, `${f} does not parse`); - const r = verify(st.program.program, { budgetMs: 3000 }); + const r = verify(st.program.program, NO_CLOCK); const all = r.units.flatMap((u) => u.obligations); for (const m of marks) { const o = all.filter((x) => x.kind === m.kind && x.line === m.line); @@ -160,7 +161,7 @@ test('canary: after a program that runs to its end, ASSERT 1 == 2 is never prove const line = lines.length + 1; const withCanary = studyOf(`${lines.join('\n')}\nASSERT 1 == 2\n`); if (!withCanary.ok) continue; - const r = verify(withCanary.program.program, { budgetMs: 3000 }); + const r = verify(withCanary.program.program, NO_CLOCK); const main = r.units.find((u) => u.kind === 'main'); const o = main?.obligations.find((x) => x.kind === 'assert' && x.line === line); if (!o) continue; @@ -174,7 +175,7 @@ test('canary: after a program that runs to its end, ASSERT 1 == 2 is never prove test('a FUNCTION that RETURNS an ARRAY of written lengths hands back those lengths: known at the call, and checked in the run', () => { // known at the call: the store into G has nothing left to prove (15-game-of-life showed it open) const life = fs.readFileSync(path.join(CORPUS_DIR, '15-game-of-life.pseudo'), 'utf8'); - const r = verify(studyOf(life).program.program, { budgetMs: 3000 }); + const r = verify(studyOf(life).program.program, NO_CLOCK); assert.deepEqual(r.units.flatMap((u) => u.obligations).filter((o) => o.status !== 'proved').map((o) => `${o.kind}@${o.line} ${o.label}`), []); // and the fact is one a run keeps: a function that would hand back another length stops at its RETURN const wrong = 'FUNCTION Two(N : INTEGER) RETURNS ARRAY[2] OF INTEGER\n DECLARE L : LIST OF INTEGER\n FOR I <- 0 TO N\n APPEND(L, I)\n NEXT I\n RETURN L\nENDFUNCTION\nDECLARE A : ARRAY[2] OF INTEGER\nA <- Two(3)\nASSERT LENGTH(A) == 2\n'; @@ -182,6 +183,6 @@ test('a FUNCTION that RETURNS an ARRAY of written lengths hands back those lengt assert.ok(st.ok, st.diagnostics.map((d) => d.message).join('; ')); const run = runPseudo(st, {}); assert.equal(`${run.error?.kind}@${run.error?.line}`, 'value@6', run.error?.message); - const obs = verify(st.program.program, { budgetMs: 3000 }).units.flatMap((u) => u.obligations); + const obs = verify(st.program.program, NO_CLOCK).units.flatMap((u) => u.obligations); assert.ok(obs.some((o) => o.line === 6 && o.status !== 'proved'), 'the RETURN of a list of unknown length is not proved to have 2 positions'); }); diff --git a/tests/pseudo-robustness.test.mjs b/tests/pseudo-robustness.test.mjs index a058095..8d79c60 100644 --- a/tests/pseudo-robustness.test.mjs +++ b/tests/pseudo-robustness.test.mjs @@ -12,6 +12,7 @@ import { pseudoToMatho } from '../dist/sync/pseudo-to-matho.js'; import { SyncHub } from '../dist/sync/hub.js'; import { checkProperties } from '../dist/engines/pseudo/testing.js'; import { run as runProgram } from '../dist/engines/pseudo/interp.js'; +import { within, NO_CLOCK } from './helpers/pace.mjs'; const errors = (src) => studyOf(src).diagnostics.filter((d) => d.severity === 'error').map((d) => d.id); const run = (src, opts = {}) => { @@ -130,9 +131,7 @@ test('the positions quantifiers visit are counted: one inside another cannot run }); test('CHOOSE with a huge K ends at the first step that cannot fit', () => { - const t0 = performance.now(); - assert.equal(run('OUTPUT CHOOSE(9007199254740991, 4503599627370495)').error?.kind, 'overflow'); - assert.ok(performance.now() - t0 < 2000); + within(2000, 'CHOOSE with a huge K', () => assert.equal(run('OUTPUT CHOOSE(9007199254740991, 4503599627370495)').error?.kind, 'overflow')); }); test('an error of the engine underneath is a stop of the run, never an exception from it', () => { @@ -170,20 +169,20 @@ test('the inputs provePseudo tries on a slow routine are held to a time budget', assert.equal(checkProperties(studyOf('FUNCTION G(N : INTEGER) RETURNS INTEGER\n RETURN N\nENDFUNCTION\n').program.program, 'G', { count: 20, budgetMs: 5000 }).cutShort, undefined); }); -/** Milliseconds `fn` takes. */ -const timed = (fn) => { const t0 = performance.now(); fn(); return performance.now() - t0; }; +// The limits below are processor time at this machine's pace (tests/helpers/pace.mjs): each was seconds or minutes +// before its fix, and is a small part of its limit after it, on any machine. test('a long program is studied in time that grows with its length, not its square (B7)', () => { // each case took seconds when the data flow was solved one statement per pass, over sets of every definition const declares = Array.from({ length: 1000 }, (_, i) => `DECLARE V${i} : INTEGER <- ${i}\n`).join(''); - assert.ok(timed(() => assert.ok(studyOf(declares).ok)) < 2000, '1000 declarations'); - assert.ok(timed(() => new SyncHub({ pseudo: '' }).edit('pseudo', declares)) < 4000, 'an edit of 1000 declarations'); + within(2000, '1000 declarations', () => assert.ok(studyOf(declares).ok)); + within(4000, 'an edit of 1000 declarations', () => new SyncHub({ pseudo: '' }).edit('pseudo', declares)); const nested = `${Array.from({ length: 150 }, (_, i) => `FOR I${i} <- 0 TO 1\n`).join('')}OUTPUT 1\n${Array.from({ length: 150 }, (_, i) => `NEXT I${149 - i}\n`).join('')}`; - assert.ok(timed(() => assert.ok(studyOf(nested).ok)) < 2000, '150 nested FOR loops'); + within(2000, '150 nested FOR loops', () => assert.ok(studyOf(nested).ok)); const arms = 2000; const chain = `DECLARE X : INTEGER <- 1\nIF X == 0 THEN\n OUTPUT X\n${Array.from({ length: arms }, (_, i) => `ELSEIF X == ${i + 1} THEN\n OUTPUT X\n`).join('')}ENDIF\n`; let st; - assert.ok(timed(() => { st = studyOf(chain); }) < 2000, `an ELSEIF chain of ${arms} arms`); + within(2000, `an ELSEIF chain of ${arms} arms`, () => { st = studyOf(chain); }); // and the answer is the whole one: X has its value in every arm, however far down the chain assert.deepEqual(st.diagnostics.filter((d) => /unset/.test(d.id)).map((d) => d.line), []); }); @@ -192,13 +191,13 @@ test('English for a text of many routines takes time that grows with their numbe const funcs = (n) => Array.from({ length: n }, (_, i) => `FUNCTION F${i}(X : INTEGER) RETURNS INTEGER\n RETURN X + ${i}\nENDFUNCTION\n`).join(''); // the English reader takes 200 routines and phrases at most (docs/limits.md): up to there the text is said in full let r; - assert.ok(timed(() => { r = pseudoToLingo(funcs(200)); }) < 1000, '200 functions'); + within(1000, '200 functions', () => { r = pseudoToLingo(funcs(200)); }); assert.ok(!('unsupported' in r)); // past it the answer says why, and still comes at once: 2,000 functions took a minute when each was held against every other - assert.ok(timed(() => { r = pseudoToLingo(funcs(600)); }) < 3000, '600 functions'); + within(3000, '600 functions', () => { r = pseudoToLingo(funcs(600)); }); assert.match(JSON.stringify(r.unsupported), /more than 200 routines and phrases/); // 2,000 of them are 6,000 lines, past what a pseudocode text is read to (docs/limits.md), and say that instead - assert.ok(timed(() => { r = pseudoToLingo(funcs(2000)); }) < 3000, '2000 functions'); + within(3000, '2000 functions', () => { r = pseudoToLingo(funcs(2000)); }); assert.match(JSON.stringify(r.unsupported), /more than 2,000 lines long/); }); @@ -207,7 +206,7 @@ test('a step costs the same at any call depth: frames that did not change are no // 390 calls deep, then a loop of 12,000 passes in the deepest call const src = 'FUNCTION F(N : INTEGER, D : INTEGER) RETURNS INTEGER\n DECLARE T : INTEGER <- 0\n IF D < 390 THEN\n RETURN F(N, D + 1)\n ENDIF\n FOR I <- 0 TO 12000\n T <- T + 1\n NEXT I\n RETURN T\nENDFUNCTION\nOUTPUT F(1, 0)\n'; let tr; - assert.ok(timed(() => { tr = run(src); }) < 1500, '20,000 steps 391 frames deep'); + within(1500, '20,000 steps 391 frames deep', () => { tr = run(src); }); assert.equal(tr.error?.kind, 'limit'); const [a, b] = [tr.states[tr.count - 2], tr.states[tr.count - 1]]; assert.equal(a.frames.length, 392); @@ -269,7 +268,7 @@ test('an ARRAY parameter and an ARRAY RETURN keep their declared lengths, as a s assert.deepEqual(run(`${first}OUTPUT FIRST([7, 8, 9])`).output, ['10']); assert.deepEqual(run('FUNCTION MAKE(N : INTEGER) RETURNS ARRAY[3] OF INTEGER\n DECLARE B : ARRAY[N] OF INTEGER\n B[0] <- 1\n RETURN B\nENDFUNCTION\nOUTPUT LENGTH(MAKE(3))').output, ['3']); // and the prover knows a parameter's lengths, and asks for them where the text does not show them - const pr = provePseudo(studyOf(`${first}DECLARE L : LIST OF INTEGER\nAPPEND(L, 7)\nOUTPUT FIRST(L)`)); + const pr = provePseudo(studyOf(`${first}DECLARE L : LIST OF INTEGER\nAPPEND(L, 7)\nOUTPUT FIRST(L)`), NO_CLOCK); const obs = pr.units.flatMap((u) => u.obligations); assert.equal(obs.find((o) => o.kind === 'index' && o.line === 2)?.status, 'proved'); assert.notEqual(obs.find((o) => o.kind === 'argument' && o.line === 6)?.status, 'proved'); diff --git a/tests/reader-limits.test.mjs b/tests/reader-limits.test.mjs index b89ce78..37e32ab 100644 --- a/tests/reader-limits.test.mjs +++ b/tests/reader-limits.test.mjs @@ -12,6 +12,7 @@ import { parseAsm, runAsmProgram } from '../dist/services/asm.js'; import { SERVICES } from '../dist/services/index.js'; import { SyncHub, LINKED, FAULT_ID } from '../dist/sync/hub.js'; import { Workspace } from '../dist/common/workspace.js'; +import { within } from './helpers/pace.mjs'; const ids = (r) => r.diagnostics.filter((d) => d.severity === 'error').map((d) => d.id); /** The app's path for a pane: a workspace opening it, and an edit through the hub. */ @@ -93,10 +94,11 @@ test('matho: an equation nested past 200 levels is matho-too-deep, and one past test('speco: a claim of 30,000 nested bars reads in linear time (each bar trimmed and searched the whole claim)', { timeout: 10000 }, () => { const big = '|{ K ∈ [0, X) : '.repeat(30000) + 'TRUE' + ' }|'.repeat(30000); - const t0 = performance.now(); - const doc = parseSpec(`SPEC F(X)\n ENSURE result = ${big}`); - assert.equal(doc.units[0].ensures.length, 1); - assert.ok(performance.now() - t0 < 5000); + // processor time at this machine's pace (tests/helpers/pace.mjs) + within(5000, '30,000 nested bars', () => { + const doc = parseSpec(`SPEC F(X)\n ENSURE result = ${big}`); + assert.equal(doc.units[0].ensures.length, 1); + }); toMath(toAscii(big)); // and what it reads is what it read before assert.equal(toAscii('|{ K ∈ [0, |T|) : T[K] = B }|'), 'LENGTH({ K IN [0, LENGTH(T)) : T[K] == B })'); diff --git a/tests/services.test.mjs b/tests/services.test.mjs index b54594a..b1688f2 100644 --- a/tests/services.test.mjs +++ b/tests/services.test.mjs @@ -9,6 +9,7 @@ import { readLingo, runLingo } from '../dist/services/lingo.js'; import { parseAsm, runAsmProgram, parseSetup } from '../dist/services/asm.js'; import { Workspace } from '../dist/common/workspace.js'; import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; +import { NO_CLOCK } from './helpers/pace.mjs'; test('every catalog problem parses, runs, and (where it has pseudocode) proves without throwing', () => { @@ -37,6 +38,7 @@ test('the off-by-one bug is caught by the run, the tests, and the proof', () => const s = new Workspace(st); assert.equal(s.trace().error.kind, 'index'); assert.ok(s.tests().some((t) => t.failures.length > 0)); + s.proofBudgetMs = Infinity; // the counted budgets' answer, the same on every machine assert.ok(s.proof().report.counts.broken >= 1); }); @@ -57,7 +59,7 @@ test('speco: a spec written separately weaves in, proves, and does not duplicate assert.match(w.source, /FOR I <- 0 TO N\n\s+INVARIANT Total == LENGTH/); const st = studyOf(w.source); assert.ok(st.ok, st.diagnostics.map((d) => d.message).join('; ')); - const pr = provePseudo(st); + const pr = provePseudo(st, NO_CLOCK); assert.equal(pr.counts.open + pr.counts.broken, 0, JSON.stringify(pr.counts)); assert.ok(pr.counts.proved >= 3); // weaving the same spec again adds nothing diff --git a/tests/sync-hub.test.mjs b/tests/sync-hub.test.mjs index 8e10760..ec9f4e8 100644 --- a/tests/sync-hub.test.mjs +++ b/tests/sync-hub.test.mjs @@ -11,6 +11,7 @@ import { pseudoToLingo } from '../dist/sync/pseudo-to-lingo.js'; import { checkMojo } from '../dist/mojo/check.js'; import { studyOf } from '../dist/services/pseudo.js'; import { EQUATIONS, FIXTURES } from './helpers/sync-programs.mjs'; +import { measure, over } from './helpers/pace.mjs'; import { Workspace } from '../dist/common/workspace.js'; import { PROBLEMS, problemById, workForProblem } from '../dist/common/problems.js'; @@ -232,13 +233,10 @@ test('timing: a full cascade (lingo edit -> pseudo, matho, speco, Mojo) on the l const start = filled.sources.lingo ? filled.sources : { lingo }; const hub = () => new SyncHub(start, undefined, { call: big.st.call }); hub().edit('lingo', lingo); // warm up the engines - const times = []; - for (let i = 0; i < 7; i++) { - const t0 = performance.now(); - hub().edit('lingo', lingo + (i % 2 ? '' : '\n')); - times.push(performance.now() - t0); - } - times.sort((a, b) => a - b); - const median = times[3]; - assert.ok(median < 50, `${big.id}: the median cascade took ${median.toFixed(1)} ms (limit 50)`); + // the median of seven, in processor time at this machine's pace (tests/helpers/pace.mjs) + const runs = []; + for (let i = 0; i < 7; i++) runs.push(measure(() => hub().edit('lingo', lingo + (i % 2 ? '' : '\n')))); + runs.sort((a, b) => a.ms - b.ms); + const failed = over({ ms: runs[3].ms, pace: Math.max(...runs.map((m) => m.pace)) }, 50, `${big.id}: the median cascade`); + assert.ok(!failed, failed ?? ''); });